{"level":"info","ts":"2026-07-10T10:26:53.931Z","logger":"setup","msg":"Loaded feature gates","featureGates":{"RayClusterStatusConditions":true,"RayJobDeletionPolicy":false}}
{"level":"info","ts":"2026-07-10T10:26:53.931Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."}
{"level":"info","ts":"2026-07-10T10:26:53.931Z","logger":"setup","msg":"Setup manager"}
{"level":"info","ts":"2026-07-10T10:26:54.225Z","logger":"setup","msg":"NetworkPolicy controller registered (annotation-based activation)"}
{"level":"info","ts":"2026-07-10T10:26:54.225Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-10T10:26:54.225Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-10T10:26:54.225Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","msg":"starting server","name":"health probe","addr":"[::]:8082"}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":false}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-07-10T10:26:54.226Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-07-10T10:26:54.526Z","msg":"attempting to acquire leader lease opendatahub/ray-operator-leader..."}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","msg":"successfully acquired lease opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.NetworkPolicy"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","msg":"Starting EventSource","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-07-10T10:26:54.536Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.535Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","msg":"Starting Controller","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","msg":"Starting workers","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","logger":"controllers.RayCluster","msg":"Starting Controller"}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","logger":"controllers.RayService","msg":"Starting Controller"}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","logger":"controllers.RayJob","msg":"Starting Controller"}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","msg":"Starting Controller","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","msg":"Starting workers","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","msg":"Starting Controller","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-07-10T10:26:54.937Z","msg":"Starting workers","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-07-10T10:29:28.348Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.348Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-4x89m"}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"92169476-9331-4d80-b964-79d680f4903e","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961"}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961"}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.361Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.366Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.366Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-4x89m"}
{"level":"info","ts":"2026-07-10T10:29:28.425Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","name":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:28.433Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","name":"raycluster-auth-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:29:28.433Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"92169476-9331-4d80-b964-79d680f4903e","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.433Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4b3aeeb9-130c-4f1b-a366-b23e0b4e924a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.525Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.531Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4b3aeeb9-130c-4f1b-a366-b23e0b4e924a","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:29:28.625Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.725Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4b3aeeb9-130c-4f1b-a366-b23e0b4e924a","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:29:28.725Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","name":"kuberay-gateway-access","namespace":"test-ns-4x89m","operation":"created"}
{"level":"info","ts":"2026-07-10T10:29:28.725Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4b3aeeb9-130c-4f1b-a366-b23e0b4e924a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.726Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.737Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:29:28.737Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.746Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.757Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:29:28.825Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:29:28.825Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"833d8466-adcf-4c03-8ef2-da3c23237815"}
{"level":"info","ts":"2026-07-10T10:29:28.825Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"61d22f4a-78e2-45cb-9061-a468edf6b339","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:28.826Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"61d22f4a-78e2-45cb-9061-a468edf6b339"}
{"level":"info","ts":"2026-07-10T10:29:28.835Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","name":"test-ns-4x89m-raycluster-auth","operation":"created"}
{"level":"info","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:29:29.526Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4x89m","name":"raycluster-auth"}}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961"}
{"level":"error","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5d01100e-e6b3-4ca5-9abe-ecf4fd6e7961","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929"}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929"}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.534Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.625Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:29:29.625Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:29:29.725Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929"}
{"level":"error","ts":"2026-07-10T10:29:29.725Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f0245074-5811-4300-9401-1a8de605f929","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.725Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"69d733be-60d4-4707-b5cd-308dc5cb7533","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f55f174d-82a2-4993-ae6b-f1fab0c4f575","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"69d733be-60d4-4707-b5cd-308dc5cb7533"}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227"}
{"level":"error","ts":"2026-07-10T10:29:29.726Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1bb73bed-8bcc-43a7-8736-159e011dc227","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363"}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363"}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363"}
{"level":"error","ts":"2026-07-10T10:29:29.736Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"8c8e9760-b6a8-4244-804e-6564d64cd363","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.749Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f55f174d-82a2-4993-ae6b-f1fab0c4f575","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121"}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121"}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121"}
{"level":"error","ts":"2026-07-10T10:29:29.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"60b4a793-9484-4c88-a913-272fbbf0d121","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.835Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f55f174d-82a2-4993-ae6b-f1fab0c4f575","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:29:29.835Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f55f174d-82a2-4993-ae6b-f1fab0c4f575","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:29.905Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e"}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e"}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e"}
{"level":"error","ts":"2026-07-10T10:29:29.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"1935d5d9-cb0d-4812-838c-5cd75e08de1e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3"}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3"}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3"}
{"level":"error","ts":"2026-07-10T10:29:30.067Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2cfb59f7-b685-49a5-b0bf-dc8a407fcbc3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.342Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","name":"kube-rbac-proxy-config-raycluster-auth","operation":"created"}
{"level":"error","ts":"2026-07-10T10:29:30.354Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.354Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12"}
{"level":"error","ts":"2026-07-10T10:29:30.354Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"741a7053-24d0-4b69-8c99-418c01510a12","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.354Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"9a8d1489-3c50-408d-9fb8-928eb42211b2","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:29:30.354Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"9a8d1489-3c50-408d-9fb8-928eb42211b2","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.354Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"9a8d1489-3c50-408d-9fb8-928eb42211b2","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.369Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"9a8d1489-3c50-408d-9fb8-928eb42211b2","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.369Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fab1e84f-c4f7-4461-9b1b-b1bcc8327034","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:29:30.369Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fab1e84f-c4f7-4461-9b1b-b1bcc8327034","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.369Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fab1e84f-c4f7-4461-9b1b-b1bcc8327034","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-10T10:29:30.378Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fab1e84f-c4f7-4461-9b1b-b1bcc8327034","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.378Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fab1e84f-c4f7-4461-9b1b-b1bcc8327034"}
{"level":"error","ts":"2026-07-10T10:29:30.378Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fab1e84f-c4f7-4461-9b1b-b1bcc8327034","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.383Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9f9ec96-3ec9-4642-803f-a7a6491eb291","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:29:30.383Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9f9ec96-3ec9-4642-803f-a7a6491eb291","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.383Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9f9ec96-3ec9-4642-803f-a7a6491eb291","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e"}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e"}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e"}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"Waiting for authentication resources to be ready","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","condition":"nil","reason":"AuthenticationPending","conditionStale":false,"observedGeneration":0,"currentGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"error","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:29:30Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}]}
{"level":"info","ts":"2026-07-10T10:29:30.388Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:29:30Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-07-10T10:29:30.392Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9f9ec96-3ec9-4642-803f-a7a6491eb291","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.392Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9f9ec96-3ec9-4642-803f-a7a6491eb291"}
{"level":"error","ts":"2026-07-10T10:29:30.392Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9f9ec96-3ec9-4642-803f-a7a6491eb291","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.397Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4x89m","name":"raycluster-auth"}}
{"level":"info","ts":"2026-07-10T10:29:30.397Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e"}
{"level":"error","ts":"2026-07-10T10:29:30.397Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"e6da2fac-a1b0-4f67-9a5b-9ac5747c484e","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.403Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"eb37dfd2-5582-48bc-974b-707d01ea31fa","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:29:30.403Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"eb37dfd2-5582-48bc-974b-707d01ea31fa","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.403Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"eb37dfd2-5582-48bc-974b-707d01ea31fa","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-10T10:29:30.425Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"eb37dfd2-5582-48bc-974b-707d01ea31fa","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.425Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"eb37dfd2-5582-48bc-974b-707d01ea31fa"}
{"level":"error","ts":"2026-07-10T10:29:30.425Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"eb37dfd2-5582-48bc-974b-707d01ea31fa","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:30.425Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f74242ef-03ab-4514-80f1-dc8535aa69e2","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c994a010-db31-41ca-939a-0ddef90ad35c","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.425Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f74242ef-03ab-4514-80f1-dc8535aa69e2"}
{"level":"info","ts":"2026-07-10T10:29:30.439Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c994a010-db31-41ca-939a-0ddef90ad35c","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:29:30.445Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"afc62dce-92f8-46b7-8ac2-20967797013a","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:29:30.445Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"afc62dce-92f8-46b7-8ac2-20967797013a","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.445Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"afc62dce-92f8-46b7-8ac2-20967797013a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.456Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c994a010-db31-41ca-939a-0ddef90ad35c","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:29:30.456Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c994a010-db31-41ca-939a-0ddef90ad35c","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:30.458Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"afc62dce-92f8-46b7-8ac2-20967797013a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:29:31.038Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","enabled":true}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","name":"raycluster-auth-head-gfzn7"}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328"}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:29:31.055Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","enabled":true}
{"level":"info","ts":"2026-07-10T10:29:31.074Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","name":"raycluster-auth-small-group-worker-l5g6r"}
{"level":"info","ts":"2026-07-10T10:29:31.074Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.074Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.074Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-10T10:29:31.074Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a628bd12-1cde-40fa-85e2-8365c3357328","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"29d36449-15ea-4429-9322-609395b5dd4f","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"41d9236a-8dac-44b8-b784-55fd02e25e11","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.087Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"29d36449-15ea-4429-9322-609395b5dd4f","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4x89m","name":"raycluster-auth"}}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867"}
{"level":"error","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"98812417-cdf3-4df8-9073-e0c6f15d7867","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.114Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"41d9236a-8dac-44b8-b784-55fd02e25e11","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0944b626-fde6-4071-a9e0-1c4edab56175","seconds":300}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:29:31.125Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"211fcc57-38ae-43aa-9610-158495c8d24c","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:29:31.126Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"41d9236a-8dac-44b8-b784-55fd02e25e11","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:29:31.126Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"41d9236a-8dac-44b8-b784-55fd02e25e11","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.136Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"75a9936c-8116-403c-8aed-8a8c3767ea5f","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.136Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"6ca5a972-b8af-4a40-bb93-6ac4240477f1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.136Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"75a9936c-8116-403c-8aed-8a8c3767ea5f","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:29:31.146Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"6ca5a972-b8af-4a40-bb93-6ac4240477f1","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:29:31.156Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"6ca5a972-b8af-4a40-bb93-6ac4240477f1","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:29:31.156Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"6ca5a972-b8af-4a40-bb93-6ac4240477f1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:29:31.520Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b037b7de-5383-45e2-811a-99537ecf0fd6","seconds":300}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:29:31.526Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"600cffae-dd8b-4428-9a3f-409b7fa69e62","seconds":300}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:29:33.088Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"6d418186-f446-4fae-b2ed-52ff64f40890","seconds":300}
{"level":"info","ts":"2026-07-10T10:29:58.826Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"ba025342-988e-4425-8836-0ffc26f0ffc5","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:29:58.826Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"ba025342-988e-4425-8836-0ffc26f0ffc5","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:30:58.827Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d5409b0f-1a65-4922-b68d-60863212a298","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:30:58.827Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d5409b0f-1a65-4922-b68d-60863212a298","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:31:58.828Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"9a756b39-a04d-4f31-83d8-c9644838403c","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:31:58.828Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"9a756b39-a04d-4f31-83d8-c9644838403c","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:32:58.829Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"cea034ab-54b0-4b15-af00-c0eae72d6da1","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:32:58.829Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"cea034ab-54b0-4b15-af00-c0eae72d6da1","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:11.694Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9297986a-e817-4aef-bdad-39e6fdc3c845","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:12.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2"}
{"level":"info","ts":"2026-07-10T10:33:12.412Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2"}
{"level":"info","ts":"2026-07-10T10:33:12.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"newHead":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-10T10:33:12.413Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9d1a3311-f4c8-43ff-9773-362f9e7d8cf2","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:29:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:12.427Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"bcbe1a80-d12c-4450-8204-46c77eeffcd9","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:12.427Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"8f933b73-246c-4d38-b152-559d75296dba","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:12.427Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"bcbe1a80-d12c-4450-8204-46c77eeffcd9","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:12.427Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"bcbe1a80-d12c-4450-8204-46c77eeffcd9","pod":"raycluster-auth-small-group-worker-l5g6r","ip":"10.133.0.31"}
{"level":"info","ts":"2026-07-10T10:33:12.450Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"8f933b73-246c-4d38-b152-559d75296dba","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:33:12.460Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"bcbe1a80-d12c-4450-8204-46c77eeffcd9","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-4x89m.svc","raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local"],"ipAddresses":["10.132.0.22","10.133.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:33:12.463Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"8f933b73-246c-4d38-b152-559d75296dba","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:33:12.463Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"8f933b73-246c-4d38-b152-559d75296dba","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:12.475Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"bcbe1a80-d12c-4450-8204-46c77eeffcd9","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-4x89m.svc","raycluster-auth-worker-svc.test-ns-4x89m.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-4x89m.svc","*.raycluster-auth-worker-svc.test-ns-4x89m.svc.cluster.local","*-worker-*.test-ns-4x89m.svc","*-worker-*.test-ns-4x89m.svc.cluster.local"],"ipAddresses":["10.132.0.22","10.133.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:33:12.475Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"bcbe1a80-d12c-4450-8204-46c77eeffcd9","rayCluster":"raycluster-auth","podIPs":["10.132.0.22","10.133.0.31"]}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:14.428Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a3e91b22-f42d-4147-ab49-95e584e1aa97","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:16.710Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"77d2edf7-35c8-4552-840e-f93503db8b03","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:17.447Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"a5606e4b-94da-4aed-bb07-3bb2beaf2c6e","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","podName":"raycluster-auth-head-gfzn7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:17.720Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"705556a6-953e-4f83-aed2-a86821b5e958","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:21.456Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532"}
{"level":"info","ts":"2026-07-10T10:33:21.456Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532"}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:21.457Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:21.525Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:21.525Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:33:21.525Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"030eb489-ec7f-4373-9d20-6d7b49e29532","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:21.537Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7e5dc23d-afb7-4d71-85bb-21021e218c57","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:21.537Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7978743b-675c-4c5f-b953-21237c128944","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:21.538Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7978743b-675c-4c5f-b953-21237c128944","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:21.538Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7978743b-675c-4c5f-b953-21237c128944","pod":"raycluster-auth-small-group-worker-l5g6r","ip":"10.133.0.31"}
{"level":"info","ts":"2026-07-10T10:33:21.538Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7978743b-675c-4c5f-b953-21237c128944","rayCluster":"raycluster-auth","podIPs":["10.132.0.22","10.133.0.31"]}
{"level":"info","ts":"2026-07-10T10:33:21.635Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7e5dc23d-afb7-4d71-85bb-21021e218c57","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:33:21.648Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7e5dc23d-afb7-4d71-85bb-21021e218c57","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:33:21.648Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"7e5dc23d-afb7-4d71-85bb-21021e218c57","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78"}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78"}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:23.538Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:23.539Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:23.539Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d4b0ddef-bba5-44ea-9169-45af0ffe1d78","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca"}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca"}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:26.770Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"82519990-a11e-41fb-912f-2c0b32ce4fca","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02"}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02"}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:27.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:27.765Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:33:27.765Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:33:27.765Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:33:27.765Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"5e6ec1de-64a6-4585-9707-fe48b9c24b02","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:27.777Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"21cae622-c294-449a-bfeb-d09c13f572a0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:27.777Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"dc421755-91e8-4224-bdda-97abad618912","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:27.777Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"21cae622-c294-449a-bfeb-d09c13f572a0","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:27.777Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"21cae622-c294-449a-bfeb-d09c13f572a0","pod":"raycluster-auth-small-group-worker-l5g6r","ip":"10.133.0.31"}
{"level":"info","ts":"2026-07-10T10:33:27.778Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"21cae622-c294-449a-bfeb-d09c13f572a0","rayCluster":"raycluster-auth","podIPs":["10.132.0.22","10.133.0.31"]}
{"level":"info","ts":"2026-07-10T10:33:27.796Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"dc421755-91e8-4224-bdda-97abad618912","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:33:27.807Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"dc421755-91e8-4224-bdda-97abad618912","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:33:27.807Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"dc421755-91e8-4224-bdda-97abad618912","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485"}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485"}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","podName":"raycluster-auth-head-gfzn7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:29.778Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"85c72e55-7161-4955-a6b4-d9a3883e6485","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7"}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7"}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:33:32.491Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2ba10f41-e2ba-4151-9d09-ec384c0b0db7","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:32.505Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"25c8cb5a-44a8-4119-8cc9-e3f90f132b29","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:32.505Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"35ff01ec-dc65-4728-bb9f-763290d64bbf","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:32.505Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"25c8cb5a-44a8-4119-8cc9-e3f90f132b29","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:32.505Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"25c8cb5a-44a8-4119-8cc9-e3f90f132b29","pod":"raycluster-auth-small-group-worker-l5g6r","ip":"10.133.0.31"}
{"level":"info","ts":"2026-07-10T10:33:32.505Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"25c8cb5a-44a8-4119-8cc9-e3f90f132b29","rayCluster":"raycluster-auth","podIPs":["10.132.0.22","10.133.0.31"]}
{"level":"info","ts":"2026-07-10T10:33:32.520Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"35ff01ec-dc65-4728-bb9f-763290d64bbf","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:33:32.531Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"35ff01ec-dc65-4728-bb9f-763290d64bbf","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:33:32.531Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"35ff01ec-dc65-4728-bb9f-763290d64bbf","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:34.506Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb"}
{"level":"info","ts":"2026-07-10T10:33:34.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb"}
{"level":"info","ts":"2026-07-10T10:33:34.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:29:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:34.507Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"79f30845-4d4e-4a03-ae3c-c82756465bbb","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c"}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c"}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:33:38.790Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"0844eab3-4aed-4822-a361-04ad7362d54c","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:38Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:38.805Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d9a990a1-c0e2-48c9-84fd-a57138cb1685","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:38.805Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"73c4aa8c-4288-4a0e-b247-0820b275559b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:38.805Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d9a990a1-c0e2-48c9-84fd-a57138cb1685","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:38.805Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d9a990a1-c0e2-48c9-84fd-a57138cb1685","pod":"raycluster-auth-small-group-worker-l5g6r","ip":"10.133.0.31"}
{"level":"info","ts":"2026-07-10T10:33:38.805Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d9a990a1-c0e2-48c9-84fd-a57138cb1685","rayCluster":"raycluster-auth","podIPs":["10.132.0.22","10.133.0.31"]}
{"level":"info","ts":"2026-07-10T10:33:38.834Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"73c4aa8c-4288-4a0e-b247-0820b275559b","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-10T10:33:38.866Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"73c4aa8c-4288-4a0e-b247-0820b275559b","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-10T10:33:38.866Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"73c4aa8c-4288-4a0e-b247-0820b275559b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:40.034Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.034Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.045Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"50b42e44-8d3b-4f11-9f48-2cb872b8dc9f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.050Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.050Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:33:40.052Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","name":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.052Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.052Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.052Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.053Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.053Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.053Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.053Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:33:40.053Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:40.058Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"50b42e44-8d3b-4f11-9f48-2cb872b8dc9f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b7695557-4903-41c9-8e2a-5dacc11fec27","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-xxj65","name":"raycluster"}}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4"}
{"level":"error","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b11eaf51-d360-4e28-9a8f-f845a89e19a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.126Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.126Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.126Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:33:40.126Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:40.130Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","name":"raycluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6"}
{"level":"error","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a3fe55d7-f493-4f00-b5d8-58c932241ab6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0"}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0"}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:33:40.136Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:40.138Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","name":"kuberay-gateway-access","namespace":"test-ns-xxj65","operation":"created"}
{"level":"info","ts":"2026-07-10T10:33:40.138Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:33:40.138Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.139Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b7695557-4903-41c9-8e2a-5dacc11fec27","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:33:40.143Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-xxj65","name":"raycluster"}}
{"level":"info","ts":"2026-07-10T10:33:40.144Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0"}
{"level":"error","ts":"2026-07-10T10:33:40.144Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6bda341b-20bf-4531-86bb-45251b0255c0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72"}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72"}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72"}
{"level":"error","ts":"2026-07-10T10:33:40.147Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"37b2263b-f166-4d24-a9d5-e4158bcdfc72","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","name":"test-ns-xxj65-raycluster","operation":"created"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7"}
{"level":"info","ts":"2026-07-10T10:33:40.225Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:33:40.225Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"38617b40-9d67-417e-a3af-868302558ba7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.232Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b7695557-4903-41c9-8e2a-5dacc11fec27","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:33:40.232Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b7695557-4903-41c9-8e2a-5dacc11fec27","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.232Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","name":"kube-rbac-proxy-config-raycluster","operation":"created"}
{"level":"info","ts":"2026-07-10T10:33:40.232Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"447358cd-4667-4f9d-acc7-265b9fc3431c","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.234Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:33:40.240Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.240Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8"}
{"level":"error","ts":"2026-07-10T10:33:40.240Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f82b298-0fa0-4d38-bb26-dc476cb257e8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.240Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2d00502d-045b-40b5-ad1d-d7ef22457817","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:33:40.240Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2d00502d-045b-40b5-ad1d-d7ef22457817","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.240Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2d00502d-045b-40b5-ad1d-d7ef22457817","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.243Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"447358cd-4667-4f9d-acc7-265b9fc3431c","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:33:40.246Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","rayCluster":"raycluster","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:33:40.246Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.246Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476"}
{"level":"error","ts":"2026-07-10T10:33:40.246Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"11cd2448-0187-451b-aee5-549185725476","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.251Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2d00502d-045b-40b5-ad1d-d7ef22457817","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.251Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c037a4c5-4dce-4ed5-b4a5-ebb0edb4e3fe","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:33:40.251Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c037a4c5-4dce-4ed5-b4a5-ebb0edb4e3fe","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.251Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c037a4c5-4dce-4ed5-b4a5-ebb0edb4e3fe","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0f3f92bd-ec6c-4973-aa87-ed5ad0addd40","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0f3f92bd-ec6c-4973-aa87-ed5ad0addd40"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9df19020-bc6f-48ea-987e-96c79e58107c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"447358cd-4667-4f9d-acc7-265b9fc3431c","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9df19020-bc6f-48ea-987e-96c79e58107c"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"447358cd-4667-4f9d-acc7-265b9fc3431c","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-07-10T10:33:40.325Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c037a4c5-4dce-4ed5-b4a5-ebb0edb4e3fe","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c037a4c5-4dce-4ed5-b4a5-ebb0edb4e3fe"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"71863a40-fe52-49b6-84ab-47cda1f36162","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:33:40.325Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c037a4c5-4dce-4ed5-b4a5-ebb0edb4e3fe","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1"}
{"level":"error","ts":"2026-07-10T10:33:40.325Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"36cfeeee-f9c4-468b-8eb8-c3a3552101f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.330Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0751e098-3b07-4722-a372-b40a7615dd12","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:33:40.330Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0751e098-3b07-4722-a372-b40a7615dd12","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.330Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0751e098-3b07-4722-a372-b40a7615dd12","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.339Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"71863a40-fe52-49b6-84ab-47cda1f36162","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:33:40.340Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0751e098-3b07-4722-a372-b40a7615dd12","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.349Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"71863a40-fe52-49b6-84ab-47cda1f36162","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:33:40.350Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"71863a40-fe52-49b6-84ab-47cda1f36162","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830"}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830"}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830"}
{"level":"error","ts":"2026-07-10T10:33:40.486Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"505e8d5e-ab9f-4018-a10a-9d40ce727830","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516"}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516"}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:40.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:40.806Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:40.806Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c43b25db-8cce-4c73-a34f-aab17f817516","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49"}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49"}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49"}
{"level":"error","ts":"2026-07-10T10:33:40.807Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"93cefd61-30e3-4f32-9455-d9f971718d49","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:41.447Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5"}
{"level":"info","ts":"2026-07-10T10:33:41.447Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5"}
{"level":"info","ts":"2026-07-10T10:33:41.447Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:41.447Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-10T10:33:41.447Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:41.448Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-10T10:33:41.448Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:41.448Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:41.448Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:41.448Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5"}
{"level":"error","ts":"2026-07-10T10:33:41.448Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1450cf22-e5ab-420f-9d70-463daf5bd9c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:33:42.728Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","enabled":true}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","name":"raycluster-head-j5zbf"}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878"}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:42.749Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","enabled":true}
{"level":"info","ts":"2026-07-10T10:33:42.825Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","name":"raycluster-small-group-worker-8gjkr"}
{"level":"info","ts":"2026-07-10T10:33:42.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:42.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:42.826Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-10T10:33:42.826Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"28f59c44-9022-4411-bee7-79e27abb5878","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e29cd690-2fab-4ff9-bee4-27ddef2e2337","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8e879bbd-61f1-46b3-95a3-f062687ebea8","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-10T10:33:42.937Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:42.938Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8e879bbd-61f1-46b3-95a3-f062687ebea8","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:33:42.944Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-xxj65","name":"raycluster"}}
{"level":"info","ts":"2026-07-10T10:33:42.944Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612"}
{"level":"error","ts":"2026-07-10T10:33:42.944Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"44fd2358-4db8-4991-9865-0870a6a1a612","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:42.949Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e29cd690-2fab-4ff9-bee4-27ddef2e2337","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:33:42.950Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"841a0373-942b-4037-8b43-62e437a439f1","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:43.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e29cd690-2fab-4ff9-bee4-27ddef2e2337","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:33:43.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e29cd690-2fab-4ff9-bee4-27ddef2e2337","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:43.025Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6346de5f-3c50-47b0-810f-c44af00b3a09","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:43.025Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"1a2370e3-9d9e-44b8-9433-cee638a58658","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:43.025Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"1a2370e3-9d9e-44b8-9433-cee638a58658","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:33:43.039Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6346de5f-3c50-47b0-810f-c44af00b3a09","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:33:43.050Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6346de5f-3c50-47b0-810f-c44af00b3a09","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:33:43.050Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6346de5f-3c50-47b0-810f-c44af00b3a09","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:43.210Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8c6beb66-200f-445e-9988-28c480a23088","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:43.374Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"964b971d-ddfe-4bf6-b91b-16749cdd18f3","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:43.835Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e83a4e0d-5312-4696-8c40-38c79a8c5e95","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e"}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e"}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:45.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.018Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:45.018Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"ac2d8d28-ecc2-4185-ae7c-e076bd9de43e","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.023Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2"}
{"level":"info","ts":"2026-07-10T10:33:45.023Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2"}
{"level":"info","ts":"2026-07-10T10:33:45.023Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:45.024Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"53d73c3e-9be1-4b43-9144-3451782344c2","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.025Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490"}
{"level":"info","ts":"2026-07-10T10:33:45.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490"}
{"level":"info","ts":"2026-07-10T10:33:45.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:45.026Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"466c9a66-1be0-4a52-a7f7-80d76e1e8490","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.042Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31"}
{"level":"info","ts":"2026-07-10T10:33:45.042Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31"}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:45.043Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"96e9048d-7b4a-4837-985b-8cf1a73c3f31","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b"}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b"}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-l5g6r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:45.046Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"b85746d7-10ac-4152-88bd-72e65ee2965b","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5"}
{"level":"info","ts":"2026-07-10T10:33:45.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5"}
{"level":"info","ts":"2026-07-10T10:33:45.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.532Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","worker Pod":"raycluster-auth-small-group-worker-l5g6r","shouldDelete":true,"reason":"The worker Pod raycluster-auth-small-group-worker-l5g6r status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T10:33:45.535Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.536Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-07-10T10:33:45.536Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.536Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:38Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:45Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:45.548Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5"}
{"level":"error","ts":"2026-07-10T10:33:45.548Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c2aa8ca4-3d06-4dd3-8d7c-3029adfc55d5","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.548Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9d4a107-14ec-48af-b159-889e5f3fd5d3","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:45.549Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9d4a107-14ec-48af-b159-889e5f3fd5d3","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:45.549Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9d4a107-14ec-48af-b159-889e5f3fd5d3","pod":"raycluster-auth-small-group-worker-l5g6r","ip":"10.133.0.31"}
{"level":"info","ts":"2026-07-10T10:33:45.549Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fc0175c6-f2dd-4dd6-b041-9c4bf5a6f8c6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:45.549Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"b9d4a107-14ec-48af-b159-889e5f3fd5d3","rayCluster":"raycluster-auth","podIPs":["10.132.0.22","10.133.0.31"]}
{"level":"info","ts":"2026-07-10T10:33:45.553Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f"}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f"}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:45.554Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"2615bd2c-e851-47d4-aee5-54cd99cf596f","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.555Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"fc0175c6-f2dd-4dd6-b041-9c4bf5a6f8c6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:45.845Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb"}
{"level":"info","ts":"2026-07-10T10:33:45.845Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb"}
{"level":"info","ts":"2026-07-10T10:33:45.845Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:33:45.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb","podName":"raycluster-auth-small-group-worker-l5g6r","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:33:45.846Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:33:45.846Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bb60d24d-f539-4cf2-905c-9a50ce9ff7cb","seconds":300}
{"level":"info","ts":"2026-07-10T10:33:45.848Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f"}
{"level":"info","ts":"2026-07-10T10:33:45.848Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f"}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f"}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.849Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:45.853Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.853Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:45.853Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:33:45.854Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:45Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:45Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:45.865Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f"}
{"level":"error","ts":"2026-07-10T10:33:45.865Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"411fca9b-2c94-4d09-8bd2-a4469a90b73f","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.865Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c7cf6be1-a878-4420-abdc-52ba7c2efff0","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:45.865Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d198b4cb-70e5-4267-b18d-9a8615c80169","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:45.865Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d198b4cb-70e5-4267-b18d-9a8615c80169","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c7cf6be1-a878-4420-abdc-52ba7c2efff0","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.871Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:45.875Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.875Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:45.875Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173"}
{"level":"error","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"d03c7464-9089-477a-b1d4-c39889172173","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef"}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef"}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.878Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d198b4cb-70e5-4267-b18d-9a8615c80169","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-4x89m.svc","raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local"],"ipAddresses":["10.132.0.22","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef"}
{"level":"info","ts":"2026-07-10T10:33:45.925Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:45.926Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.926Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.926Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:45.929Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.929Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:45.930Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef"}
{"level":"error","ts":"2026-07-10T10:33:45.930Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"f549a12c-2335-4e56-8e09-d9d56b2d18ef","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.938Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d198b4cb-70e5-4267-b18d-9a8615c80169","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-4x89m.svc","raycluster-auth-worker-svc.test-ns-4x89m.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-4x89m.svc","*.raycluster-auth-worker-svc.test-ns-4x89m.svc.cluster.local","*-worker-*.test-ns-4x89m.svc","*-worker-*.test-ns-4x89m.svc.cluster.local"],"ipAddresses":["10.132.0.22","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:33:45.938Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d198b4cb-70e5-4267-b18d-9a8615c80169"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745"}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:45.950Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.951Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:45.951Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:45.954Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:45.954Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:45.955Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745"}
{"level":"error","ts":"2026-07-10T10:33:45.955Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"110d1373-0a3d-43b7-84b4-e52f9bc90745","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4"}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.025Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:46.030Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.030Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:46.031Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4"}
{"level":"error","ts":"2026-07-10T10:33:46.031Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"c4783b35-e562-4ea6-95bd-622a983cb0e4","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-gfzn7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b"}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.112Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:46.116Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.117Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:46.117Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b"}
{"level":"error","ts":"2026-07-10T10:33:46.117Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"fd072c65-ebf7-4d8f-9865-22112f50148b","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.191Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de"}
{"level":"info","ts":"2026-07-10T10:33:46.191Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de"}
{"level":"info","ts":"2026-07-10T10:33:46.192Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:46.192Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","Found 1 head Pod":"raycluster-auth-head-gfzn7","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-10 10:33:17 +0000 UTC,FinishedAt:2026-07-10 10:33:45 +0000 UTC,ContainerID:cri-o://9ddfbf6a1f1b318d364eb82879191b0cc13dc7d006e6d6cd182c4414769e3acd,}"}
{"level":"info","ts":"2026-07-10T10:33:46.192Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","head Pod":"raycluster-auth-head-gfzn7","shouldDelete":true,"reason":"The head Pod raycluster-auth-head-gfzn7 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T10:33:46.199Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-auth-head-gfzn7 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.199Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","error":"The head Pod raycluster-auth-head-gfzn7 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-10T10:33:46.199Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:33:46.199Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:45Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:33:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:46Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:46.216Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"a432f660-458b-4878-8a14-3c2aecf3bf72","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.217Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"a432f660-458b-4878-8a14-3c2aecf3bf72","pod":"raycluster-auth-head-gfzn7","ip":"10.132.0.22"}
{"level":"info","ts":"2026-07-10T10:33:46.217Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"a432f660-458b-4878-8a14-3c2aecf3bf72"}
{"level":"info","ts":"2026-07-10T10:33:46.216Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f5a41a3b-d76f-424d-9049-45613bc24865","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.217Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de"}
{"level":"error","ts":"2026-07-10T10:33:46.217Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"16f6783b-a297-4bd6-9a55-4ee95c71a5de","error":"The head Pod raycluster-auth-head-gfzn7 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.224Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f5a41a3b-d76f-424d-9049-45613bc24865","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.277Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6"}
{"level":"info","ts":"2026-07-10T10:33:46.277Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6"}
{"level":"info","ts":"2026-07-10T10:33:46.277Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6"}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.278Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:46.282Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.282Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:46.282Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6"}
{"level":"error","ts":"2026-07-10T10:33:46.283Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"9714c69e-f4c6-45c1-bfd4-0ddb8cdb5fb6","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.553Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c"}
{"level":"info","ts":"2026-07-10T10:33:46.553Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c"}
{"level":"info","ts":"2026-07-10T10:33:46.553Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c"}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:33:46.554Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:46.558Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.558Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:46.558Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.558Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.558Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.558Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"newHead":{"serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-10T10:33:46.558Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:46Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"raycluster-auth-head-gfzn7","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:46Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:46.571Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c"}
{"level":"error","ts":"2026-07-10T10:33:46.571Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"bd82caac-6bfa-4ce4-8c44-43f9bff0b18c","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.571Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67"}
{"level":"info","ts":"2026-07-10T10:33:46.571Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"66052194-6bc5-4fe0-84e9-8caf8fee05e4","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:33:46.571Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"38d0ce8b-3e07-43b7-b5a0-ced3a764914e","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:33:46.572Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:46.576Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.576Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:46.576Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.576Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.576Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.576Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:33:46.576Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:46Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:46Z","stateTransitionTimes":{"ready":"2026-07-10T10:33:38Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:46Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:33:38Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:29:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:29:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:33:45Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:33:46.578Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"66052194-6bc5-4fe0-84e9-8caf8fee05e4","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.588Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"38d0ce8b-3e07-43b7-b5a0-ced3a764914e","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-4x89m.svc","raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:33:46.626Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67"}
{"level":"error","ts":"2026-07-10T10:33:46.626Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"11b6aef2-e2ed-4d16-bfe1-9f723b793e67","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.626Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"26702e36-cb13-457b-b0cf-22e6d21e5311","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.632Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"26702e36-cb13-457b-b0cf-22e6d21e5311","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.639Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"38d0ce8b-3e07-43b7-b5a0-ced3a764914e","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-4x89m.svc","raycluster-auth-worker-svc.test-ns-4x89m.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-4x89m.svc","*.raycluster-auth-worker-svc.test-ns-4x89m.svc.cluster.local","*-worker-*.test-ns-4x89m.svc","*-worker-*.test-ns-4x89m.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:33:46.639Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"38d0ce8b-3e07-43b7-b5a0-ced3a764914e"}
{"level":"info","ts":"2026-07-10T10:33:46.639Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c11019f0-49a0-4b81-9173-10ff68db71a8","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.639Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c11019f0-49a0-4b81-9173-10ff68db71a8"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:33:46.923Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-4x89m.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:33:46.924Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","enabled":true}
{"level":"error","ts":"2026-07-10T10:33:46.928Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:46.928Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:33:46.928Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.928Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.928Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","filter labels":["test-ns-4x89m",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:33:46.928Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db"}
{"level":"error","ts":"2026-07-10T10:33:46.928Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"75cafec8-09d2-4a42-af28-0b37935a69db","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-4x89m because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"a4f866b1-7a40-4f14-99ad-922a92ec6c6d"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"dbd5b6c2-17e1-4736-b14d-7588c9548795","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"reconcileID":"7019f073-cc18-4d9c-bddc-9dc499b81cee"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"dbd5b6c2-17e1-4736-b14d-7588c9548795"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","namespace":"test-ns-4x89m","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:33:50.198Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:33:50.204Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","name":"test-ns-4x89m-raycluster-auth","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:33:50.211Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","configMap":"kube-rbac-proxy-config-raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.216Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","serviceAccount":"raycluster-auth-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:33:50.222Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.222Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-4x89m"}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"d34be3af-e3b5-49f6-a5e5-c9c0d57a7232","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7","namespace":"test-ns-4x89m","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f8229ae0-16f1-4096-8092-26da460eb3a1"}
{"level":"info","ts":"2026-07-10T10:33:50.232Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f2a822dc-f388-4c54-976c-c5d44f9bc035","namespace":"test-ns-4x89m","clusterName":"raycluster-auth"}
{"level":"error","ts":"2026-07-10T10:33:50.235Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7","error":"rayclusters.ray.io \"raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:50.236Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7"}
{"level":"error","ts":"2026-07-10T10:33:50.236Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"c1386657-467a-4782-8d05-554aff89ecd7","error":"rayclusters.ray.io \"raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:50.236Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4cb683e0-7830-45cc-94f6-b8b2963d4239","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:33:50.236Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4cb683e0-7830-45cc-94f6-b8b2963d4239"}
{"level":"info","ts":"2026-07-10T10:33:50.236Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4cb683e0-7830-45cc-94f6-b8b2963d4239","namespace":"test-ns-4x89m","authClusters":0}
{"level":"info","ts":"2026-07-10T10:33:50.236Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4cb683e0-7830-45cc-94f6-b8b2963d4239","namespace":"test-ns-4x89m"}
{"level":"info","ts":"2026-07-10T10:33:50.238Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"0e23f749-893b-49af-92f9-2478b0ed0f9f"}
{"level":"info","ts":"2026-07-10T10:33:50.238Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"377ea00e-0bb1-42cf-82ac-b154432226e2","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:33:50.238Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"377ea00e-0bb1-42cf-82ac-b154432226e2"}
{"level":"info","ts":"2026-07-10T10:33:50.238Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"377ea00e-0bb1-42cf-82ac-b154432226e2","namespace":"test-ns-4x89m","authClusters":0}
{"level":"info","ts":"2026-07-10T10:33:50.238Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"377ea00e-0bb1-42cf-82ac-b154432226e2","namespace":"test-ns-4x89m"}
{"level":"error","ts":"2026-07-10T10:33:50.240Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f2a822dc-f388-4c54-976c-c5d44f9bc035","issuer":"ray-selfsigned-issuer-raycluster-auth","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:50.241Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"74769a07-5c76-43f8-97ed-882e4f68cd8a","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-4x89m"}}
{"level":"info","ts":"2026-07-10T10:33:50.241Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"74769a07-5c76-43f8-97ed-882e4f68cd8a"}
{"level":"info","ts":"2026-07-10T10:33:50.241Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"74769a07-5c76-43f8-97ed-882e4f68cd8a","namespace":"test-ns-4x89m","authClusters":0}
{"level":"info","ts":"2026-07-10T10:33:50.241Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"74769a07-5c76-43f8-97ed-882e4f68cd8a","namespace":"test-ns-4x89m"}
{"level":"error","ts":"2026-07-10T10:33:50.243Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f2a822dc-f388-4c54-976c-c5d44f9bc035","certificate":"ray-worker-cert-raycluster-auth","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:50.245Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"4cf16526-59e2-435e-96d2-13b2fd7b8af7"}
{"level":"error","ts":"2026-07-10T10:33:50.252Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f2a822dc-f388-4c54-976c-c5d44f9bc035","certificate":"ray-ca-certificate-raycluster-auth","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:50.325Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f2a822dc-f388-4c54-976c-c5d44f9bc035","errorCount":3}
{"level":"info","ts":"2026-07-10T10:33:50.325Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f2a822dc-f388-4c54-976c-c5d44f9bc035"}
{"level":"error","ts":"2026-07-10T10:33:50.325Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"f2a822dc-f388-4c54-976c-c5d44f9bc035","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:33:50.425Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"800a7ada-7931-4682-b7cc-94bb4876a616","namespace":"test-ns-4x89m","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:33:50.425Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-4x89m"},"namespace":"test-ns-4x89m","name":"raycluster-auth","reconcileID":"800a7ada-7931-4682-b7cc-94bb4876a616","namespace":"test-ns-4x89m","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-07-10T10:34:10.326Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6c7bfb21-74aa-418c-9ba6-33913349f083","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:34:10.326Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6c7bfb21-74aa-418c-9ba6-33913349f083","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:34:10.340Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6c7bfb21-74aa-418c-9ba6-33913349f083","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-xxj65.svc","raycluster-head-svc.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:34:10.353Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6c7bfb21-74aa-418c-9ba6-33913349f083","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-xxj65.svc","raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*.raycluster-worker-svc.test-ns-xxj65.svc","*.raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*-worker-*.test-ns-xxj65.svc","*-worker-*.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:34:10.353Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6c7bfb21-74aa-418c-9ba6-33913349f083","rayCluster":"raycluster","podIPs":["10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:35:09.215Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ca688f22-d932-412f-9273-2de490c46bc4","seconds":300}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:35:10.231Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"6306e31e-7b83-4b57-ac44-9ced41d5a253","seconds":300}
{"level":"info","ts":"2026-07-10T10:35:10.354Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"eb7b6801-5e43-47e4-acc7-3de620fc25d3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:35:10.354Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"eb7b6801-5e43-47e4-acc7-3de620fc25d3","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:35:10.354Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"eb7b6801-5e43-47e4-acc7-3de620fc25d3","rayCluster":"raycluster","podIPs":["10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:36:10.355Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"df85cb1b-f1ec-468d-a007-cf0b6073dfbb","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:36:10.355Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"df85cb1b-f1ec-468d-a007-cf0b6073dfbb","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:36:10.355Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"df85cb1b-f1ec-468d-a007-cf0b6073dfbb","rayCluster":"raycluster","podIPs":["10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:37:10.357Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b5d87fc5-40bf-4a15-b624-1ebc3c0858f6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:10.357Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b5d87fc5-40bf-4a15-b624-1ebc3c0858f6","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:37:10.357Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b5d87fc5-40bf-4a15-b624-1ebc3c0858f6","rayCluster":"raycluster","podIPs":["10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"newHead":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-10T10:37:29.292Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c142c2eb-394c-4c5b-bd06-163edf6123c7","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:33:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:29.304Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33ccc4c9-3dd8-4096-b68d-824ec375aaae","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:29.304Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33ccc4c9-3dd8-4096-b68d-824ec375aaae","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:37:29.304Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33ccc4c9-3dd8-4096-b68d-824ec375aaae","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:37:29.304Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"1f3d5baa-4ffb-4bb2-bf39-7e858555e12c","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:29.324Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"1f3d5baa-4ffb-4bb2-bf39-7e858555e12c","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:37:29.331Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33ccc4c9-3dd8-4096-b68d-824ec375aaae","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-xxj65.svc","raycluster-head-svc.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["10.134.0.17","10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:37:29.339Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"1f3d5baa-4ffb-4bb2-bf39-7e858555e12c","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:37:29.339Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"1f3d5baa-4ffb-4bb2-bf39-7e858555e12c","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:29.343Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33ccc4c9-3dd8-4096-b68d-824ec375aaae","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-xxj65.svc","raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*.raycluster-worker-svc.test-ns-xxj65.svc","*.raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*-worker-*.test-ns-xxj65.svc","*-worker-*.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["10.134.0.17","10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:37:29.343Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33ccc4c9-3dd8-4096-b68d-824ec375aaae","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8"}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8"}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:31.304Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:37:31.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:37:31.305Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:37:31.305Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2352ab4e-d942-44a3-a637-a18fc96d6ad8","seconds":300}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","podName":"raycluster-head-j5zbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:37:34.298Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a8f2f02d-2c58-47d4-b1d4-5b7def0fe9d8","seconds":300}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776"}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776"}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:38.321Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:38.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:38.322Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:37:38.322Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"0dc1f844-2256-45c6-9b7a-97d1eccb8776","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:38.340Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2cd343b7-ead2-4814-a153-5345e71eca73","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:38.340Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2cd343b7-ead2-4814-a153-5345e71eca73","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:37:38.340Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2cd343b7-ead2-4814-a153-5345e71eca73","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:37:38.340Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2cd343b7-ead2-4814-a153-5345e71eca73","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:37:38.341Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e45a9bce-a509-4e4d-ad48-7183a2fd63eb","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:38.353Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e45a9bce-a509-4e4d-ad48-7183a2fd63eb","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:37:38.367Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e45a9bce-a509-4e4d-ad48-7183a2fd63eb","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:37:38.367Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e45a9bce-a509-4e4d-ad48-7183a2fd63eb","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51"}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51"}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:37:40.344Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"fd91f08c-bd48-4c95-9a55-d9a1704add51","seconds":300}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315"}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315"}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:37:45.877Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"e6d4f493-5d6e-48a8-b954-308cf5893315","seconds":300}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205"}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205"}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:37:46.886Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3ff969f0-d941-49fb-8fec-5fd138421205","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:46.899Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"04758066-422c-4255-a15d-be4c2c8817e6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:46.899Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2637e63-0edd-456b-8882-8bfb7487918f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:46.899Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"04758066-422c-4255-a15d-be4c2c8817e6","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:37:46.899Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"04758066-422c-4255-a15d-be4c2c8817e6","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:37:46.899Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"04758066-422c-4255-a15d-be4c2c8817e6","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:37:46.912Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2637e63-0edd-456b-8882-8bfb7487918f","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:37:46.923Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2637e63-0edd-456b-8882-8bfb7487918f","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:37:46.923Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2637e63-0edd-456b-8882-8bfb7487918f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5"}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5"}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","podName":"raycluster-head-j5zbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:37:48.900Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"049fdd51-4c50-4745-a038-23c57d4ffec5","seconds":300}
{"level":"info","ts":"2026-07-10T10:37:49.349Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b"}
{"level":"info","ts":"2026-07-10T10:37:49.349Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b"}
{"level":"info","ts":"2026-07-10T10:37:49.349Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","podName":"raycluster-small-group-worker-8gjkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","podName":"raycluster-small-group-worker-8gjkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:37:49.350Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"04965819-bc62-4010-94cd-6484c3f8c32b","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:49.362Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"28b1ccc7-5370-45a2-b3ab-2873e8b7702d","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:49.362Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d0242340-5360-4e5e-bb69-9858bb32409b","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:49.362Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d0242340-5360-4e5e-bb69-9858bb32409b","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:37:49.362Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d0242340-5360-4e5e-bb69-9858bb32409b","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:37:49.362Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d0242340-5360-4e5e-bb69-9858bb32409b","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:37:49.372Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"28b1ccc7-5370-45a2-b3ab-2873e8b7702d","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:37:49.384Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"28b1ccc7-5370-45a2-b3ab-2873e8b7702d","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:37:49.384Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"28b1ccc7-5370-45a2-b3ab-2873e8b7702d","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:51.362Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465"}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465"}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","podName":"raycluster-small-group-worker-8gjkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","podName":"raycluster-small-group-worker-8gjkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:33:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:37:51.363Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"dd564688-e006-439f-b4c0-dbecd9abc465","seconds":300}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121"}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121"}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:37:57.910Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"9e6e55c2-71c7-436e-967c-00dc0d37f121","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:57.923Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f9b998b0-1ec4-4f42-b94e-122586423daf","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:57.923Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2b9be24-4608-490d-9322-deb859e3bb4d","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:57.923Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f9b998b0-1ec4-4f42-b94e-122586423daf","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:37:57.923Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f9b998b0-1ec4-4f42-b94e-122586423daf","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:37:57.923Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f9b998b0-1ec4-4f42-b94e-122586423daf","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:37:57.940Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2b9be24-4608-490d-9322-deb859e3bb4d","name":"raycluster-head"}
{"level":"info","ts":"2026-07-10T10:37:57.956Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2b9be24-4608-490d-9322-deb859e3bb4d","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-10T10:37:57.956Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e2b9be24-4608-490d-9322-deb859e3bb4d","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:37:58.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2de0ab1e-8814-41cd-b29f-b574a5d927ae","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:37:58.530Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2de0ab1e-8814-41cd-b29f-b574a5d927ae","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:37:58.542Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2de0ab1e-8814-41cd-b29f-b574a5d927ae"}
{"level":"info","ts":"2026-07-10T10:37:58.543Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2de0ab1e-8814-41cd-b29f-b574a5d927ae","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:37:58.543Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2de0ab1e-8814-41cd-b29f-b574a5d927ae","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:37:58.543Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2de0ab1e-8814-41cd-b29f-b574a5d927ae","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:37:58.553Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"92f11a91-7c38-4c3c-9e9a-d1d64531514f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:37:58.553Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"92f11a91-7c38-4c3c-9e9a-d1d64531514f","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.553Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"92f11a91-7c38-4c3c-9e9a-d1d64531514f","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.553Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"92f11a91-7c38-4c3c-9e9a-d1d64531514f","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.570Z","logger":"raycluster-resource","msg":"default","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.570Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-z4gdw","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:37:58.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"92f11a91-7c38-4c3c-9e9a-d1d64531514f","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"850e6319-8fd2-4c8c-95f0-9e103ee12cdd","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f"}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f"}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","mode":"IntegratedOAuth","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.580Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.585Z","logger":"raycluster-resource","msg":"default","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.586Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-z4gdw","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:37:58.605Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.605Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.605Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"850e6319-8fd2-4c8c-95f0-9e103ee12cdd","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.589Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","name":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:58.625Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:58.625Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"fa80e84f-172c-4d08-b7f9-1b943099c157","cluster":"counter-z4gdw"}
{"level":"error","ts":"2026-07-10T10:37:58.625Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.625Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:58.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.625Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","name":"counter-z4gdw-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.625Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"92f11a91-7c38-4c3c-9e9a-d1d64531514f"}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-z4gdw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:37:58.625Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"92f11a91-7c38-4c3c-9e9a-d1d64531514f","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","name":"counter-z4gdw","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.641Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"fa80e84f-172c-4d08-b7f9-1b943099c157","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"1ca1cf4a-7682-430f-9a13-72e61b89da5b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"1ca1cf4a-7682-430f-9a13-72e61b89da5b","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"1ca1cf4a-7682-430f-9a13-72e61b89da5b","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"1ca1cf4a-7682-430f-9a13-72e61b89da5b"}
{"level":"info","ts":"2026-07-10T10:37:58.725Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"1ca1cf4a-7682-430f-9a13-72e61b89da5b","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","name":"counter-z4gdw","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"counter-z4gdw"}}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f"}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","name":"kuberay-gateway-access","namespace":"test-ns-7lp9t","operation":"created"}
{"level":"error","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8d0eb207-0a55-4b20-a0df-1f192806de0f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e"}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e"}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-z4gdw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","name":"counter-z4gdw","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:58.734Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.738Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"fa80e84f-172c-4d08-b7f9-1b943099c157","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:37:58.738Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"fa80e84f-172c-4d08-b7f9-1b943099c157","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.743Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","name":"test-ns-7lp9t-counter-z4gdw","operation":"created"}
{"level":"info","ts":"2026-07-10T10:37:58.746Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e"}
{"level":"error","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"65ffc412-c6be-4504-b195-b0c9d766e88e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825"}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825"}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"45456fb0-8a19-416f-a569-622fb96ca5d2","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-z4gdw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","name":"counter-z4gdw","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:37:58.747Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"df96a4f0-52bd-4d22-9e51-4ec11c9ee1b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:37:58.748Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"df96a4f0-52bd-4d22-9e51-4ec11c9ee1b6","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.748Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"df96a4f0-52bd-4d22-9e51-4ec11c9ee1b6","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.748Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"df96a4f0-52bd-4d22-9e51-4ec11c9ee1b6"}
{"level":"info","ts":"2026-07-10T10:37:58.748Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"df96a4f0-52bd-4d22-9e51-4ec11c9ee1b6","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:37:58.751Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","name":"kube-rbac-proxy-config-counter-z4gdw","operation":"created"}
{"level":"info","ts":"2026-07-10T10:37:58.754Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","name":"counter-z4gdw","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"counter-z4gdw"}}
{"level":"info","ts":"2026-07-10T10:37:58.754Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825"}
{"level":"error","ts":"2026-07-10T10:37:58.754Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"d74e3b77-000e-4a2c-a0a2-3662bce49825","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230","rayCluster":"counter-z4gdw","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"f250a17d-01e5-423f-a303-054c51cfc230"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"95d5a349-e5aa-4d98-b1a3-107c471f218d","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"95d5a349-e5aa-4d98-b1a3-107c471f218d"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9"}
{"level":"error","ts":"2026-07-10T10:37:58.757Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9"}
{"level":"error","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:37:58.757Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d9ae5ec4-cbe5-4f36-a859-29b0614ac2f9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"ffb1bac9-b4fd-415f-9828-e99af51fa9dd","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9"}
{"level":"error","ts":"2026-07-10T10:37:58.757Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"a00c4730-eb13-4275-be63-f16541ae5ad9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"ffb1bac9-b4fd-415f-9828-e99af51fa9dd","mode":"IntegratedOAuth","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.757Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"ffb1bac9-b4fd-415f-9828-e99af51fa9dd","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.759Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"45456fb0-8a19-416f-a569-622fb96ca5d2","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0"}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0"}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0"}
{"level":"error","ts":"2026-07-10T10:37:58.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"5a5227fa-15ac-4e93-8b9d-cb3798eb90c0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.836Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"45456fb0-8a19-416f-a569-622fb96ca5d2","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:37:58.836Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"45456fb0-8a19-416f-a569-622fb96ca5d2","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.836Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"ffb1bac9-b4fd-415f-9828-e99af51fa9dd","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.836Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a1ac0cbb-fc70-41fb-a4a0-8259f3fc1793","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.836Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a1ac0cbb-fc70-41fb-a4a0-8259f3fc1793","mode":"IntegratedOAuth","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.836Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a1ac0cbb-fc70-41fb-a4a0-8259f3fc1793","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"34793efb-8487-4273-8ba1-391fcdaf938e","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5776d227-5f0c-4d88-aee0-eb5fdc465e4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5776d227-5f0c-4d88-aee0-eb5fdc465e4b","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"67d9e0de-22bd-4df4-8e11-608521f18160","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5776d227-5f0c-4d88-aee0-eb5fdc465e4b","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"34793efb-8487-4273-8ba1-391fcdaf938e"}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5776d227-5f0c-4d88-aee0-eb5fdc465e4b"}
{"level":"info","ts":"2026-07-10T10:37:58.837Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5776d227-5f0c-4d88-aee0-eb5fdc465e4b","RayCluster":"counter-z4gdw","State":""}
{"level":"error","ts":"2026-07-10T10:37:58.844Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a1ac0cbb-fc70-41fb-a4a0-8259f3fc1793","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.844Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a1ac0cbb-fc70-41fb-a4a0-8259f3fc1793"}
{"level":"error","ts":"2026-07-10T10:37:58.844Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a1ac0cbb-fc70-41fb-a4a0-8259f3fc1793","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.850Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"742d4144-a5b7-4d6c-951d-911b8754be92","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:37:58.850Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"742d4144-a5b7-4d6c-951d-911b8754be92","mode":"IntegratedOAuth","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.850Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"742d4144-a5b7-4d6c-951d-911b8754be92","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"67d9e0de-22bd-4df4-8e11-608521f18160","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4"}
{"level":"error","ts":"2026-07-10T10:37:58.925Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"ce3f6798-3d86-4d94-9543-34a7190293a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:58.925Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"742d4144-a5b7-4d6c-951d-911b8754be92","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:58.940Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"67d9e0de-22bd-4df4-8e11-608521f18160","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:37:58.940Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"67d9e0de-22bd-4df4-8e11-608521f18160","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067"}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067"}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067"}
{"level":"error","ts":"2026-07-10T10:37:59.086Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"bb28c308-ad55-499f-843b-fa5a8fa62067","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d"}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d"}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d"}
{"level":"error","ts":"2026-07-10T10:37:59.407Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"75f10e4f-aaa5-4999-8c36-46532b6d4d9d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6"}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6"}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:37:59.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:37:59.924Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:37:59.924Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"27459df8-ca30-41be-a5c0-7c7def2417e6","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a"}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a"}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","error":"head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"error","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found"}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"counter-z4gdw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a"}
{"level":"error","ts":"2026-07-10T10:38:00.048Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"2946c42c-080d-499f-88d6-b8f28ac0d39a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-z4gdw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","mode":"IntegratedOAuth","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","cluster":"counter-z4gdw","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","cluster":"counter-z4gdw","authType":"IntegratedOAuth","serviceAccount":"counter-z4gdw-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-z4gdw","ray.io/group":"headgroup","ray.io/identifier":"counter-z4gdw-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local","num-cpus":"4","num-gpus":"4","resources":"'{\"R1\": 4}'"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-07-10T10:38:01.329Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","enabled":true}
{"level":"info","ts":"2026-07-10T10:38:01.349Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","name":"counter-z4gdw-head-8cs86"}
{"level":"info","ts":"2026-07-10T10:38:01.349Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.349Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.349Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-z4gdw-head-svc"},"newHead":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:38:01.349Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c637e146-a410-4d19-9cb2-09f1f178cd8b","name":"counter-z4gdw","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fe65ca3-f484-4755-b6c2-96acbefdcc88","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"cf93ed54-aee0-4c6f-8a61-309220c360c4","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-z4gdw-head-svc"},"newHead":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","name":"counter-z4gdw","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"cf93ed54-aee0-4c6f-8a61-309220c360c4","rayCluster":"counter-z4gdw","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d8d531fc-79a3-4f71-9139-dede61fca5b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d8d531fc-79a3-4f71-9139-dede61fca5b1","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d8d531fc-79a3-4f71-9139-dede61fca5b1","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d8d531fc-79a3-4f71-9139-dede61fca5b1"}
{"level":"info","ts":"2026-07-10T10:38:01.363Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d8d531fc-79a3-4f71-9139-dede61fca5b1","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","name":"counter-z4gdw","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"counter-z4gdw"}}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113"}
{"level":"error","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c6597d96-b7be-4be8-9a0d-04349e9ef113","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-z4gdw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb"}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb"}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:01.371Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"36a60c80-e3ee-456b-8311-5ef90d5910fb","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:01.375Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fe65ca3-f484-4755-b6c2-96acbefdcc88","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:38:01.376Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426"}
{"level":"info","ts":"2026-07-10T10:38:01.376Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426"}
{"level":"info","ts":"2026-07-10T10:38:01.376Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:01.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:01.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:01.377Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.377Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.377Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:01.377Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"980fef25-793f-4022-957a-2b9fc430e426","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:01.387Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fe65ca3-f484-4755-b6c2-96acbefdcc88","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:38:01.387Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fe65ca3-f484-4755-b6c2-96acbefdcc88","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.391Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b"}
{"level":"info","ts":"2026-07-10T10:38:01.391Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b"}
{"level":"info","ts":"2026-07-10T10:38:01.391Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:01.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:01.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:01.392Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.392Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.392Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:38:01.392Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"8cf0b565-3dc9-4238-9725-0e04e22d469b","name":"counter-z4gdw","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"097663ec-7ec6-4cff-b850-59d0a5f00963","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"fa0b43b0-e242-4a14-ac72-890e6ce45ffa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"fa0b43b0-e242-4a14-ac72-890e6ce45ffa","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"fa0b43b0-e242-4a14-ac72-890e6ce45ffa","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d110af0b-5863-4d5a-b702-4953389040f4","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"fa0b43b0-e242-4a14-ac72-890e6ce45ffa"}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"fa0b43b0-e242-4a14-ac72-890e6ce45ffa","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:01.425Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"097663ec-7ec6-4cff-b850-59d0a5f00963","rayCluster":"counter-z4gdw","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:38:01.438Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d110af0b-5863-4d5a-b702-4953389040f4","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:38:01.454Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d110af0b-5863-4d5a-b702-4953389040f4","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:38:01.454Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d110af0b-5863-4d5a-b702-4953389040f4","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.553Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"13917051-985e-4781-b1ed-e936eca89ad3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:01.553Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"13917051-985e-4781-b1ed-e936eca89ad3","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:01.553Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"13917051-985e-4781-b1ed-e936eca89ad3","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:01.553Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"13917051-985e-4781-b1ed-e936eca89ad3"}
{"level":"info","ts":"2026-07-10T10:38:01.553Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"13917051-985e-4781-b1ed-e936eca89ad3","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d"}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d"}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:01.808Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"cfbf590d-9cf7-4869-8991-5a99b64c693d","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e"}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e"}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"newHead":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:38:02.525Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"9b320dc5-eff0-4e63-ac82-2188ccce1b6e","name":"counter-z4gdw","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:02.537Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d84979c2-3e35-4d9d-bbd4-7647a153e46e","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:02.538Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"323b3e1a-09f1-4e1d-a35a-b1f9f2fde5bb","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:02.538Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"323b3e1a-09f1-4e1d-a35a-b1f9f2fde5bb","pod":"counter-z4gdw-head-8cs86","ip":"10.132.0.23"}
{"level":"info","ts":"2026-07-10T10:38:02.538Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d052ba12-93a7-48eb-8db1-fe2549111b06","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:02.538Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d052ba12-93a7-48eb-8db1-fe2549111b06","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:02.538Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d052ba12-93a7-48eb-8db1-fe2549111b06","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:02.538Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d052ba12-93a7-48eb-8db1-fe2549111b06"}
{"level":"info","ts":"2026-07-10T10:38:02.538Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"d052ba12-93a7-48eb-8db1-fe2549111b06","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:02.549Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d84979c2-3e35-4d9d-bbd4-7647a153e46e","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:38:02.550Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"323b3e1a-09f1-4e1d-a35a-b1f9f2fde5bb","certificate":"ray-head-cert-counter-z4gdw","dnsNames":["counter-z4gdw-head-svc","localhost","counter-z4gdw-head-svc.test-ns-7lp9t.svc","counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.132.0.23","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:38:02.562Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d84979c2-3e35-4d9d-bbd4-7647a153e46e","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:38:02.562Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d84979c2-3e35-4d9d-bbd4-7647a153e46e","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:02.563Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"323b3e1a-09f1-4e1d-a35a-b1f9f2fde5bb","certificate":"ray-worker-cert-counter-z4gdw","dnsNames":["counter-z4gdw-worker-svc","localhost","counter-z4gdw-worker-svc.test-ns-7lp9t.svc","counter-z4gdw-worker-svc.test-ns-7lp9t.svc.cluster.local","*.counter-z4gdw-worker-svc.test-ns-7lp9t.svc","*.counter-z4gdw-worker-svc.test-ns-7lp9t.svc.cluster.local","*-worker-*.test-ns-7lp9t.svc","*-worker-*.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.132.0.23","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:38:02.563Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"323b3e1a-09f1-4e1d-a35a-b1f9f2fde5bb","rayCluster":"counter-z4gdw","podIPs":["10.132.0.23"]}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55"}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55"}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:03.403Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"1ffa9fc0-59d5-45fb-8ad8-7ffb19b5bb55","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:04.554Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"9cda0e83-cb51-4725-b461-3d180c3e6f8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:04.554Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"9cda0e83-cb51-4725-b461-3d180c3e6f8f","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:04.554Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"9cda0e83-cb51-4725-b461-3d180c3e6f8f","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:04.554Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"9cda0e83-cb51-4725-b461-3d180c3e6f8f"}
{"level":"info","ts":"2026-07-10T10:38:04.554Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"9cda0e83-cb51-4725-b461-3d180c3e6f8f","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d"}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d"}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d","podName":"counter-z4gdw-head-8cs86","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:07.549Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"dc3b9299-6f27-4b4b-a5db-b169a063007d","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:07.555Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5aa328dc-fc8b-4fd0-8e54-5ce7f20499ea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:07.555Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5aa328dc-fc8b-4fd0-8e54-5ce7f20499ea","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:07.555Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5aa328dc-fc8b-4fd0-8e54-5ce7f20499ea","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:07.555Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5aa328dc-fc8b-4fd0-8e54-5ce7f20499ea"}
{"level":"info","ts":"2026-07-10T10:38:07.555Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5aa328dc-fc8b-4fd0-8e54-5ce7f20499ea","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470"}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470"}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470","podName":"counter-z4gdw-head-8cs86","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470","podName":"counter-z4gdw-head-8cs86","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:38:08.553Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"0f21be14-2e9d-438b-a669-e0d3f29ce470","name":"counter-z4gdw","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d93cb3ae-1a7b-43bc-8955-de7f5c65f597","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"4b4e4279-c2d3-4f17-9e94-dad7a44f8bc0","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"4b4e4279-c2d3-4f17-9e94-dad7a44f8bc0","pod":"counter-z4gdw-head-8cs86","ip":"10.132.0.23"}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"68fae1e5-f959-499d-973e-468d98844d97","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"68fae1e5-f959-499d-973e-468d98844d97","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"68fae1e5-f959-499d-973e-468d98844d97","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"4b4e4279-c2d3-4f17-9e94-dad7a44f8bc0","rayCluster":"counter-z4gdw","podIPs":["10.132.0.23"]}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"68fae1e5-f959-499d-973e-468d98844d97"}
{"level":"info","ts":"2026-07-10T10:38:08.565Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"68fae1e5-f959-499d-973e-468d98844d97","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:08.575Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d93cb3ae-1a7b-43bc-8955-de7f5c65f597","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:38:08.587Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d93cb3ae-1a7b-43bc-8955-de7f5c65f597","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:38:08.587Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"d93cb3ae-1a7b-43bc-8955-de7f5c65f597","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:10.358Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d189e3b2-0584-45ec-ab25-205ec22b6798","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:38:10.358Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d189e3b2-0584-45ec-ab25-205ec22b6798","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:38:10.358Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d189e3b2-0584-45ec-ab25-205ec22b6798","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:38:10.358Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d189e3b2-0584-45ec-ab25-205ec22b6798","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:38:10.555Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5a39a8ad-8a1c-42a3-a2cd-a29edb97e01c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:10.555Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5a39a8ad-8a1c-42a3-a2cd-a29edb97e01c","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:10.555Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5a39a8ad-8a1c-42a3-a2cd-a29edb97e01c","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:10.556Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5a39a8ad-8a1c-42a3-a2cd-a29edb97e01c"}
{"level":"info","ts":"2026-07-10T10:38:10.556Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5a39a8ad-8a1c-42a3-a2cd-a29edb97e01c","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b"}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b"}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b","podName":"counter-z4gdw-head-8cs86","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b","podName":"counter-z4gdw-head-8cs86","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:10.566Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"c0e3236e-e8aa-4f10-ba44-4045f4d8b95b","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:13.556Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c05f548e-ddbf-4102-8cc9-e9faf118be40","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:13.556Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c05f548e-ddbf-4102-8cc9-e9faf118be40","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:13.557Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c05f548e-ddbf-4102-8cc9-e9faf118be40","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:13.557Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c05f548e-ddbf-4102-8cc9-e9faf118be40"}
{"level":"info","ts":"2026-07-10T10:38:13.557Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c05f548e-ddbf-4102-8cc9-e9faf118be40","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:16.557Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c064701b-7b11-4c78-9b23-55dcb88d120d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:16.557Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c064701b-7b11-4c78-9b23-55dcb88d120d","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:16.557Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c064701b-7b11-4c78-9b23-55dcb88d120d","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:16.558Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c064701b-7b11-4c78-9b23-55dcb88d120d"}
{"level":"info","ts":"2026-07-10T10:38:16.558Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"c064701b-7b11-4c78-9b23-55dcb88d120d","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:19.558Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"619eb70e-eae4-48b4-85c1-b74362f6e717","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:19.558Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"619eb70e-eae4-48b4-85c1-b74362f6e717","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:19.558Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"619eb70e-eae4-48b4-85c1-b74362f6e717","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:19.558Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"619eb70e-eae4-48b4-85c1-b74362f6e717"}
{"level":"info","ts":"2026-07-10T10:38:19.558Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"619eb70e-eae4-48b4-85c1-b74362f6e717","RayCluster":"counter-z4gdw","State":""}
{"level":"info","ts":"2026-07-10T10:38:19.603Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"86d271cf-4039-4709-ae0f-2b875271a701"}
{"level":"info","ts":"2026-07-10T10:38:19.603Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"86d271cf-4039-4709-ae0f-2b875271a701"}
{"level":"info","ts":"2026-07-10T10:38:19.603Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"86d271cf-4039-4709-ae0f-2b875271a701","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:19.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"86d271cf-4039-4709-ae0f-2b875271a701","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:19.603Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"86d271cf-4039-4709-ae0f-2b875271a701","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:19.603Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"86d271cf-4039-4709-ae0f-2b875271a701","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:38:19.603Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"86d271cf-4039-4709-ae0f-2b875271a701","name":"counter-z4gdw","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:19Z","stateTransitionTimes":{"ready":"2026-07-10T10:38:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"98357571-a99c-4095-97a4-3c1bc43d3f2e","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"0bf8e917-4b1c-4db6-a3d1-5fa002201636","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"0bf8e917-4b1c-4db6-a3d1-5fa002201636","pod":"counter-z4gdw-head-8cs86","ip":"10.132.0.23"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"0bf8e917-4b1c-4db6-a3d1-5fa002201636","rayCluster":"counter-z4gdw","podIPs":["10.132.0.23"]}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","head service name":"counter-z4gdw-head-svc","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","head service URL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","SubmissionMode":"HTTPMode","RayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","dashboardURL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:38:19.622Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b6491593-77d0-4ff6-9bc6-9e9864504ad2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:38:19.634Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"98357571-a99c-4095-97a4-3c1bc43d3f2e","name":"counter-z4gdw-head"}
{"level":"info","ts":"2026-07-10T10:38:19.635Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b5afe90a-49b8-4b1c-bd85-c302c03f0f08","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:19.635Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b5afe90a-49b8-4b1c-bd85-c302c03f0f08","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:19.635Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b5afe90a-49b8-4b1c-bd85-c302c03f0f08","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:19.635Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b5afe90a-49b8-4b1c-bd85-c302c03f0f08"}
{"level":"info","ts":"2026-07-10T10:38:19.646Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b5afe90a-49b8-4b1c-bd85-c302c03f0f08","JobId":"counter-6hnjd"}
{"level":"info","ts":"2026-07-10T10:38:19.647Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"b5afe90a-49b8-4b1c-bd85-c302c03f0f08","rayJob":"counter","jobInfo":"{\"runtime_env\":{\"env_vars\":{\"counter_name\":\"test_counter\"}},\"entrypoint_resources\":{\"R1\":2},\"entrypoint\":\"python /home/ray/jobs/counter.py\",\"submission_id\":\"counter-6hnjd\",\"entrypoint_num_cpus\":2,\"entrypoint_num_gpus\":2}"}
{"level":"info","ts":"2026-07-10T10:38:19.649Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"98357571-a99c-4095-97a4-3c1bc43d3f2e","name":"counter-z4gdw-workers"}
{"level":"info","ts":"2026-07-10T10:38:19.649Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"98357571-a99c-4095-97a4-3c1bc43d3f2e","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:21.624Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"7552da2c-88ef-4e92-9d33-dd324c06ce87"}
{"level":"info","ts":"2026-07-10T10:38:21.624Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"7552da2c-88ef-4e92-9d33-dd324c06ce87"}
{"level":"info","ts":"2026-07-10T10:38:21.624Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"7552da2c-88ef-4e92-9d33-dd324c06ce87","1 head service found":"counter-z4gdw-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:21.624Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"7552da2c-88ef-4e92-9d33-dd324c06ce87","Found 1 head Pod":"counter-z4gdw-head-8cs86","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:21.624Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"7552da2c-88ef-4e92-9d33-dd324c06ce87","head Pod":"counter-z4gdw-head-8cs86","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-z4gdw-head-8cs86. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:21.624Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"7552da2c-88ef-4e92-9d33-dd324c06ce87","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:21.624Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"7552da2c-88ef-4e92-9d33-dd324c06ce87","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:22.559Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5168d368-b8f6-4c9c-9382-d60c68dc2203","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:22.559Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5168d368-b8f6-4c9c-9382-d60c68dc2203","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:22.559Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5168d368-b8f6-4c9c-9382-d60c68dc2203","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:22.559Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5168d368-b8f6-4c9c-9382-d60c68dc2203"}
{"level":"info","ts":"2026-07-10T10:38:22.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5168d368-b8f6-4c9c-9382-d60c68dc2203","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","dashboardURL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:38:20Z"},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","dashboardURL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:19Z","stateTransitionTimes":{"ready":"2026-07-10T10:38:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:38:22.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"5168d368-b8f6-4c9c-9382-d60c68dc2203","old JobStatus":"","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:38:22.576Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"4eb4d7bf-b064-43eb-bbf0-07625e942cc7","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:22.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"4eb4d7bf-b064-43eb-bbf0-07625e942cc7","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:22.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"4eb4d7bf-b064-43eb-bbf0-07625e942cc7","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:22.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"4eb4d7bf-b064-43eb-bbf0-07625e942cc7"}
{"level":"info","ts":"2026-07-10T10:38:22.580Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"4eb4d7bf-b064-43eb-bbf0-07625e942cc7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:38:20Z"},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","dashboardURL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:19Z","stateTransitionTimes":{"ready":"2026-07-10T10:38:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:38:20Z"},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","dashboardURL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:19Z","stateTransitionTimes":{"ready":"2026-07-10T10:38:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:38:25.576Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"8eb471d5-7ed9-48d7-91b3-34b9b67ec760","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.576Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"8eb471d5-7ed9-48d7-91b3-34b9b67ec760","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.576Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"8eb471d5-7ed9-48d7-91b3-34b9b67ec760","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.576Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"8eb471d5-7ed9-48d7-91b3-34b9b67ec760"}
{"level":"info","ts":"2026-07-10T10:38:25.580Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"8eb471d5-7ed9-48d7-91b3-34b9b67ec760","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:38:20Z"},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","dashboardURL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:37:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:19Z","stateTransitionTimes":{"ready":"2026-07-10T10:38:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:38:20Z","endTime":"2026-07-10T10:38:25Z"},"jobId":"counter-6hnjd","rayClusterName":"counter-z4gdw","dashboardURL":"counter-z4gdw-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-10T10:37:58Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:19Z","stateTransitionTimes":{"ready":"2026-07-10T10:38:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"counter-z4gdw-head-8cs86","serviceName":"counter-z4gdw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:38:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:37:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:37:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:38:25.580Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"8eb471d5-7ed9-48d7-91b3-34b9b67ec760","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-10T10:38:25.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"32931a57-ad47-425e-9f18-d306b7816567","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.594Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"32931a57-ad47-425e-9f18-d306b7816567","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:38:25 +0000 UTC","Now":"2026-07-10T10:38:25.594Z","ShutdownTime":"2026-07-10T10:38:25.000Z"}
{"level":"info","ts":"2026-07-10T10:38:25.594Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"32931a57-ad47-425e-9f18-d306b7816567","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:38:25.604Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"32931a57-ad47-425e-9f18-d306b7816567","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.604Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"32931a57-ad47-425e-9f18-d306b7816567","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:38:25.604Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"32931a57-ad47-425e-9f18-d306b7816567","RayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"6846acc3-0af6-4209-9d7d-f532d57eb91a"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"497d1002-0963-4f38-aa40-88f9ee939582","rayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"123dc1a1-a645-46fd-ac62-454d6d3af7d7","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"reconcileID":"81933f8a-183f-4f4d-93c9-61f2c07cbaef"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"123dc1a1-a645-46fd-ac62-454d6d3af7d7","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:38:25 +0000 UTC","Now":"2026-07-10T10:38:25.605Z","ShutdownTime":"2026-07-10T10:38:25.000Z"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","namespace":"test-ns-7lp9t","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"123dc1a1-a645-46fd-ac62-454d6d3af7d7","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"123dc1a1-a645-46fd-ac62-454d6d3af7d7","RayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"123dc1a1-a645-46fd-ac62-454d6d3af7d7","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"123dc1a1-a645-46fd-ac62-454d6d3af7d7","RayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"497d1002-0963-4f38-aa40-88f9ee939582","pod":"counter-z4gdw-head-8cs86","ip":"10.132.0.23"}
{"level":"info","ts":"2026-07-10T10:38:25.605Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"497d1002-0963-4f38-aa40-88f9ee939582","rayCluster":"counter-z4gdw","podIPs":["10.132.0.23"]}
{"level":"info","ts":"2026-07-10T10:38:25.610Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","name":"kuberay-gateway-access","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:38:25.616Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","name":"test-ns-7lp9t-counter-z4gdw","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:38:25.620Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","configMap":"kube-rbac-proxy-config-counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.625Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","serviceAccount":"counter-z4gdw-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:38:25.632Z","logger":"raycluster-resource","msg":"default","name":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.632Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-z4gdw","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:38:25.641Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"1fdfbd98-1100-4648-8d42-5285f4325676","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.641Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.641Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.641Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8","cluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.641Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8","namespace":"test-ns-7lp9t","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:38:25.641Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"109147f6-b436-4030-b21c-153fb4fb9d37"}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a3939fca-26ff-4e54-901b-689a588b6f20","namespace":"test-ns-7lp9t","clusterName":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2df2a5e6-0b5b-46af-8061-2123662cd65d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2df2a5e6-0b5b-46af-8061-2123662cd65d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:38:25 +0000 UTC","Now":"2026-07-10T10:38:25.642Z","ShutdownTime":"2026-07-10T10:38:25.000Z"}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2df2a5e6-0b5b-46af-8061-2123662cd65d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2df2a5e6-0b5b-46af-8061-2123662cd65d","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2df2a5e6-0b5b-46af-8061-2123662cd65d","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:38:25.642Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"2df2a5e6-0b5b-46af-8061-2123662cd65d","RayCluster":"counter-z4gdw"}
{"level":"error","ts":"2026-07-10T10:38:25.647Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8","error":"rayclusters.ray.io \"counter-z4gdw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.647Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8"}
{"level":"error","ts":"2026-07-10T10:38:25.647Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"92e5b7b9-c204-4781-8b82-6914397268d8","error":"rayclusters.ray.io \"counter-z4gdw\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.647Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"b5090e5e-60be-41e9-b1da-2d1081c4bb2c","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.647Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"b5090e5e-60be-41e9-b1da-2d1081c4bb2c"}
{"level":"info","ts":"2026-07-10T10:38:25.648Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"b5090e5e-60be-41e9-b1da-2d1081c4bb2c","namespace":"test-ns-7lp9t","authClusters":0}
{"level":"info","ts":"2026-07-10T10:38:25.648Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"b5090e5e-60be-41e9-b1da-2d1081c4bb2c","namespace":"test-ns-7lp9t"}
{"level":"error","ts":"2026-07-10T10:38:25.652Z","msg":"Failed to delete CA issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a3939fca-26ff-4e54-901b-689a588b6f20","issuer":"ray-ca-issuer-counter-z4gdw","error":"issuers.cert-manager.io \"ray-ca-issuer-counter-z4gdw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:247\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.653Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"94f282a9-9340-4cc2-96ef-780b5ea64de5","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.653Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"94f282a9-9340-4cc2-96ef-780b5ea64de5"}
{"level":"info","ts":"2026-07-10T10:38:25.653Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"94f282a9-9340-4cc2-96ef-780b5ea64de5","namespace":"test-ns-7lp9t","authClusters":0}
{"level":"info","ts":"2026-07-10T10:38:25.653Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"94f282a9-9340-4cc2-96ef-780b5ea64de5","namespace":"test-ns-7lp9t"}
{"level":"error","ts":"2026-07-10T10:38:25.656Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a3939fca-26ff-4e54-901b-689a588b6f20","certificate":"ray-head-cert-counter-z4gdw","error":"certificates.cert-manager.io \"ray-head-cert-counter-z4gdw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.662Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"117ca9ca-3f3a-4a50-9ba5-d71b3af36954"}
{"level":"info","ts":"2026-07-10T10:38:25.662Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"8fd1be84-0dc9-4fd7-9c58-aed5b8b871dc"}
{"level":"info","ts":"2026-07-10T10:38:25.665Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"6bd601f4-88a4-41e1-b6db-e807541ddb83","namespacedName":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.665Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"6bd601f4-88a4-41e1-b6db-e807541ddb83"}
{"level":"info","ts":"2026-07-10T10:38:25.665Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"6bd601f4-88a4-41e1-b6db-e807541ddb83","namespace":"test-ns-7lp9t","authClusters":0}
{"level":"info","ts":"2026-07-10T10:38:25.665Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"6bd601f4-88a4-41e1-b6db-e807541ddb83","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:38:25.676Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a3939fca-26ff-4e54-901b-689a588b6f20","errorCount":2}
{"level":"info","ts":"2026-07-10T10:38:25.676Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a3939fca-26ff-4e54-901b-689a588b6f20"}
{"level":"error","ts":"2026-07-10T10:38:25.676Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"a3939fca-26ff-4e54-901b-689a588b6f20","error":"issuers.cert-manager.io \"ray-ca-issuer-counter-z4gdw\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.682Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"61310369-2efa-4506-9e39-be174c19a56a","namespace":"test-ns-7lp9t","clusterName":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.682Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"counter-z4gdw","reconcileID":"61310369-2efa-4506-9e39-be174c19a56a","namespace":"test-ns-7lp9t","clusterName":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:25.725Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9dae3253-ded5-418d-8967-ac47058c710d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.725Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9dae3253-ded5-418d-8967-ac47058c710d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:38:25.737Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9dae3253-ded5-418d-8967-ac47058c710d"}
{"level":"info","ts":"2026-07-10T10:38:25.737Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9dae3253-ded5-418d-8967-ac47058c710d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:38:25.737Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9dae3253-ded5-418d-8967-ac47058c710d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:38:25.737Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9dae3253-ded5-418d-8967-ac47058c710d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:38:25.748Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"4d405656-fbcb-4f1a-92de-95001c725227","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.748Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"4d405656-fbcb-4f1a-92de-95001c725227","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.748Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"4d405656-fbcb-4f1a-92de-95001c725227","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.748Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"4d405656-fbcb-4f1a-92de-95001c725227","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.752Z","logger":"raycluster-resource","msg":"default","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.752Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-rkvnj","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"4d405656-fbcb-4f1a-92de-95001c725227","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9c321cd5-c93a-40de-b99a-5621383c7b76","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"4d405656-fbcb-4f1a-92de-95001c725227"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"4d405656-fbcb-4f1a-92de-95001c725227","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b3405250-e8b9-4f07-b2a1-a7a08016b3d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b3405250-e8b9-4f07-b2a1-a7a08016b3d9","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b3405250-e8b9-4f07-b2a1-a7a08016b3d9","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b3405250-e8b9-4f07-b2a1-a7a08016b3d9"}
{"level":"info","ts":"2026-07-10T10:38:25.770Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b3405250-e8b9-4f07-b2a1-a7a08016b3d9","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:25.776Z","logger":"raycluster-resource","msg":"default","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.776Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-rkvnj","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","name":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:38:25.779Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","name":"fail-rkvnj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","name":"fail-rkvnj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-rkvnj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"fail-rkvnj"}}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112"}
{"level":"error","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"bbef2dde-c72a-459c-b6bd-ce661aca8112","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9c321cd5-c93a-40de-b99a-5621383c7b76","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4b7db1ce-2aa3-4abd-aeb3-becf459d7569","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","name":"fail-rkvnj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:25.825Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.826Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"f21d8d39-ac00-4ea4-a021-04418faa80fa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.826Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"f21d8d39-ac00-4ea4-a021-04418faa80fa","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.826Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"f21d8d39-ac00-4ea4-a021-04418faa80fa","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.826Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"f21d8d39-ac00-4ea4-a021-04418faa80fa"}
{"level":"info","ts":"2026-07-10T10:38:25.826Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"f21d8d39-ac00-4ea4-a021-04418faa80fa","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","name":"fail-rkvnj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-rkvnj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"fail-rkvnj"}}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29"}
{"level":"error","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"68f1ef77-3644-4fd0-a0b6-54a1afe5ab29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d"}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d"}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","name":"fail-rkvnj-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:38:25.835Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","name":"fail-rkvnj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:25.836Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:38:25.836Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.846Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4b7db1ce-2aa3-4abd-aeb3-becf459d7569","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:25.847Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","name":"kuberay-gateway-access","namespace":"test-ns-7lp9t","operation":"created"}
{"level":"info","ts":"2026-07-10T10:38:25.848Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.852Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d"}
{"level":"error","ts":"2026-07-10T10:38:25.852Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"69b7a47f-3851-487c-8e2c-7d6d13124c4d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0ee84e78-6138-439b-83a8-f5c98096f944","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0ee84e78-6138-439b-83a8-f5c98096f944","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0ee84e78-6138-439b-83a8-f5c98096f944","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47"}
{"level":"error","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"200fee7b-404a-4a98-b6c4-b5d5655bea47","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0ee84e78-6138-439b-83a8-f5c98096f944"}
{"level":"info","ts":"2026-07-10T10:38:25.853Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0ee84e78-6138-439b-83a8-f5c98096f944","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:25.856Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","name":"test-ns-7lp9t-fail-rkvnj","operation":"created"}
{"level":"info","ts":"2026-07-10T10:38:25.860Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4b7db1ce-2aa3-4abd-aeb3-becf459d7569","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:25.860Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4b7db1ce-2aa3-4abd-aeb3-becf459d7569","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.860Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3ccf208f-f1b4-429b-a63a-804ba5488254","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:25.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad"}
{"level":"info","ts":"2026-07-10T10:38:25.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad"}
{"level":"info","ts":"2026-07-10T10:38:25.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:25.925Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:25.925Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.025Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"info","ts":"2026-07-10T10:38:25.927Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","name":"kube-rbac-proxy-config-fail-rkvnj","operation":"created"}
{"level":"info","ts":"2026-07-10T10:38:26.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:25.927Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:38:26.025Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad"}
{"level":"error","ts":"2026-07-10T10:38:26.025Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7b6b61b4-736d-4483-81a0-c0562f68f1ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:25.935Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3ccf208f-f1b4-429b-a63a-804ba5488254","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:26.125Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:26.125Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-rkvnj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","rayCluster":"fail-rkvnj","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934"}
{"level":"error","ts":"2026-07-10T10:38:26.125Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"29bba553-8b81-4d79-84fe-af85d1415934","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-rkvnj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:38:26.125Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-rkvnj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f"}
{"level":"info","ts":"2026-07-10T10:38:26.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"22ed85fb-13ec-4b18-92e0-b6445e32dec6","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"error","ts":"2026-07-10T10:38:26.225Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"27984a7b-1253-441f-a431-24506c84803f","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-rkvnj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.225Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"22ed85fb-13ec-4b18-92e0-b6445e32dec6","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.225Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"22ed85fb-13ec-4b18-92e0-b6445e32dec6","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.225Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"80fbe71c-4aaf-4c62-958f-2c0815a5ae8b","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.225Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94"}
{"level":"error","ts":"2026-07-10T10:38:26.225Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a5340f59-5745-4fe5-a711-496a11d71f94","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.230Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3ccf208f-f1b4-429b-a63a-804ba5488254","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:26.230Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3ccf208f-f1b4-429b-a63a-804ba5488254","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"22ed85fb-13ec-4b18-92e0-b6445e32dec6","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b2c540b4-a6a1-4883-a352-138a306e0439","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b2c540b4-a6a1-4883-a352-138a306e0439","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b2c540b4-a6a1-4883-a352-138a306e0439","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c434d330-77e2-4d41-901c-e85cd1669372","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a604a41-fe68-4b32-bbdc-7c8a9201864b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a604a41-fe68-4b32-bbdc-7c8a9201864b","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a604a41-fe68-4b32-bbdc-7c8a9201864b","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a604a41-fe68-4b32-bbdc-7c8a9201864b"}
{"level":"info","ts":"2026-07-10T10:38:26.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a604a41-fe68-4b32-bbdc-7c8a9201864b","RayCluster":"fail-rkvnj","State":""}
{"level":"error","ts":"2026-07-10T10:38:26.240Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"80fbe71c-4aaf-4c62-958f-2c0815a5ae8b","error":"certificates.cert-manager.io \"ray-worker-cert-fail-rkvnj\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:38:26.240Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"80fbe71c-4aaf-4c62-958f-2c0815a5ae8b","error":"certificates.cert-manager.io \"ray-worker-cert-fail-rkvnj\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.240Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"80fbe71c-4aaf-4c62-958f-2c0815a5ae8b"}
{"level":"error","ts":"2026-07-10T10:38:26.240Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"80fbe71c-4aaf-4c62-958f-2c0815a5ae8b","error":"certificates.cert-manager.io \"ray-worker-cert-fail-rkvnj\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.240Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"1186ee00-642e-47f6-8e93-02bed09fe627","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.240Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"1186ee00-642e-47f6-8e93-02bed09fe627"}
{"level":"info","ts":"2026-07-10T10:38:26.246Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b2c540b4-a6a1-4883-a352-138a306e0439","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.251Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b77482ea-7034-4a5d-a30a-3fcbee50e040","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.251Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b77482ea-7034-4a5d-a30a-3fcbee50e040"}
{"level":"info","ts":"2026-07-10T10:38:26.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c434d330-77e2-4d41-901c-e85cd1669372","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:26.337Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c434d330-77e2-4d41-901c-e85cd1669372","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:26.337Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c434d330-77e2-4d41-901c-e85cd1669372","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee"}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee"}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee"}
{"level":"error","ts":"2026-07-10T10:38:26.386Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"23cf103f-27cf-46f2-9f62-4ceb4873f9ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f"}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f"}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","error":"head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"error","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found"}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:38:26.707Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f"}
{"level":"error","ts":"2026-07-10T10:38:26.707Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6412d268-9c27-495c-afad-fff516e2284f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-rkvnj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","cluster":"fail-rkvnj","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","cluster":"fail-rkvnj","authType":"IntegratedOAuth","serviceAccount":"fail-rkvnj-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-rkvnj","ray.io/group":"headgroup","ray.io/identifier":"fail-rkvnj-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:38:27.347Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","enabled":true}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","name":"fail-rkvnj-head-p58jf"}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3"}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:38:27.368Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","enabled":true}
{"level":"info","ts":"2026-07-10T10:38:27.400Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","name":"fail-rkvnj-small-group-worker-qv8sz"}
{"level":"info","ts":"2026-07-10T10:38:27.400Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.400Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.400Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-rkvnj-head-svc"},"newHead":{"podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"}}
{"level":"info","ts":"2026-07-10T10:38:27.400Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80de9593-d84f-4a14-8bdc-0c7a4eb0dfc3","name":"fail-rkvnj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"fcff3934-1dff-44fb-bbe0-e29adbae70e2","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a9d7febd-9197-41bd-90a2-ce0823591019","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a9d7febd-9197-41bd-90a2-ce0823591019","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a9d7febd-9197-41bd-90a2-ce0823591019","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e276231a-5d73-49af-ae04-0f29a9961d36","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"fcff3934-1dff-44fb-bbe0-e29adbae70e2"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a9d7febd-9197-41bd-90a2-ce0823591019"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a9d7febd-9197-41bd-90a2-ce0823591019","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:27.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:27.420Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b61d289e-15e0-4102-907d-b8849397afdc","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:27.435Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a0865134-3cac-4250-bcd8-90a16aa981bb","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:27.437Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e276231a-5d73-49af-ae04-0f29a9961d36","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:38:27.439Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4ca03f5d-0c54-451a-9156-2b8f8c04a3bf","name":"fail-rkvnj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:27.454Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8888c85-85dd-47ae-bf14-21d002ddb14e","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.454Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a0ff62b2-49cd-4f2f-8b0f-8b256c16f967","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:27.454Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a0ff62b2-49cd-4f2f-8b0f-8b256c16f967","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.454Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a0ff62b2-49cd-4f2f-8b0f-8b256c16f967","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:27.454Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8888c85-85dd-47ae-bf14-21d002ddb14e"}
{"level":"info","ts":"2026-07-10T10:38:27.454Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a0ff62b2-49cd-4f2f-8b0f-8b256c16f967"}
{"level":"info","ts":"2026-07-10T10:38:27.454Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a0ff62b2-49cd-4f2f-8b0f-8b256c16f967","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:27.457Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e276231a-5d73-49af-ae04-0f29a9961d36","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:27.457Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e276231a-5d73-49af-ae04-0f29a9961d36","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.457Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"0d979853-cfdf-408c-8e37-6fbde65115eb","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:27.458Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"b1d49374-79dd-4c3b-958f-1bfe1d35d21d","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:27.475Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"0d979853-cfdf-408c-8e37-6fbde65115eb","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:27.497Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"0d979853-cfdf-408c-8e37-6fbde65115eb","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:27.497Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"0d979853-cfdf-408c-8e37-6fbde65115eb","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:27.826Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555"}
{"level":"info","ts":"2026-07-10T10:38:27.826Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555"}
{"level":"info","ts":"2026-07-10T10:38:27.826Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:27.827Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8f1f33b6-dd4e-44b4-9d5d-a184e0975555","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:27.854Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab"}
{"level":"info","ts":"2026-07-10T10:38:27.854Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab"}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:27.855Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"99648a16-c0d0-4af6-b0f8-4d7b2f97d8ab","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:28.066Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"417a3943-3398-4f91-b26d-11d5bd736018","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:28.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"0e0e7851-171b-4c0b-aad5-e9c439c3b548","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:28.593Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"0e0e7851-171b-4c0b-aad5-e9c439c3b548","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:38:25 +0000 UTC","Now":"2026-07-10T10:38:28.593Z","ShutdownTime":"2026-07-10T10:38:25.000Z"}
{"level":"info","ts":"2026-07-10T10:38:28.593Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"0e0e7851-171b-4c0b-aad5-e9c439c3b548","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:38:28.593Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"0e0e7851-171b-4c0b-aad5-e9c439c3b548","RayCluster":{"name":"counter-z4gdw","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:28.593Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"0e0e7851-171b-4c0b-aad5-e9c439c3b548","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:38:28.593Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"0e0e7851-171b-4c0b-aad5-e9c439c3b548","RayCluster":"counter-z4gdw"}
{"level":"info","ts":"2026-07-10T10:38:28.637Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7"}
{"level":"info","ts":"2026-07-10T10:38:28.637Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7"}
{"level":"info","ts":"2026-07-10T10:38:28.637Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"newHead":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"}}
{"level":"info","ts":"2026-07-10T10:38:28.638Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d5a500bb-f0e6-4d63-a280-c85bc85122e7","name":"fail-rkvnj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:28.650Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c44b8e47-79b4-48f6-9144-dcf2bde9036d","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:28.650Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9741c75c-8043-4fd3-ba46-eb2b246421c5","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:28.651Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b9149f9d-1fd1-40f7-8999-66655d079941","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:28.651Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b9149f9d-1fd1-40f7-8999-66655d079941","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:28.651Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9741c75c-8043-4fd3-ba46-eb2b246421c5","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:38:28.651Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9741c75c-8043-4fd3-ba46-eb2b246421c5","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:38:28.651Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b9149f9d-1fd1-40f7-8999-66655d079941","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:28.651Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b9149f9d-1fd1-40f7-8999-66655d079941"}
{"level":"info","ts":"2026-07-10T10:38:28.651Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b9149f9d-1fd1-40f7-8999-66655d079941","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:28.663Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9741c75c-8043-4fd3-ba46-eb2b246421c5","certificate":"ray-head-cert-fail-rkvnj","dnsNames":["fail-rkvnj-head-svc","localhost","fail-rkvnj-head-svc.test-ns-7lp9t.svc","fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.132.0.24","10.133.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:38:28.666Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c44b8e47-79b4-48f6-9144-dcf2bde9036d","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:28.676Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9741c75c-8043-4fd3-ba46-eb2b246421c5","certificate":"ray-worker-cert-fail-rkvnj","dnsNames":["fail-rkvnj-worker-svc","localhost","fail-rkvnj-small-group","fail-rkvnj-worker-svc.test-ns-7lp9t.svc","fail-rkvnj-worker-svc.test-ns-7lp9t.svc.cluster.local","*.fail-rkvnj-worker-svc.test-ns-7lp9t.svc","*.fail-rkvnj-worker-svc.test-ns-7lp9t.svc.cluster.local","*-worker-*.test-ns-7lp9t.svc","*-worker-*.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.132.0.24","10.133.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:38:28.676Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9741c75c-8043-4fd3-ba46-eb2b246421c5"}
{"level":"info","ts":"2026-07-10T10:38:28.676Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c44b8e47-79b4-48f6-9144-dcf2bde9036d","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:28.676Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c44b8e47-79b4-48f6-9144-dcf2bde9036d","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:28.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1cecc928-7465-46c4-ad01-d476af968483","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:28.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1cecc928-7465-46c4-ad01-d476af968483","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:28.749Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1cecc928-7465-46c4-ad01-d476af968483","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:28.749Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1cecc928-7465-46c4-ad01-d476af968483"}
{"level":"info","ts":"2026-07-10T10:38:28.749Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1cecc928-7465-46c4-ad01-d476af968483","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:29.420Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009"}
{"level":"info","ts":"2026-07-10T10:38:29.420Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009"}
{"level":"info","ts":"2026-07-10T10:38:29.420Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:29.421Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"257b012b-ead9-497d-b5f1-8ef109d7a009","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:31.750Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"03d1947f-793b-4895-98bf-fca131bc7f00","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:31.750Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"03d1947f-793b-4895-98bf-fca131bc7f00","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:31.750Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"03d1947f-793b-4895-98bf-fca131bc7f00","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:31.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"03d1947f-793b-4895-98bf-fca131bc7f00"}
{"level":"info","ts":"2026-07-10T10:38:31.750Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"03d1947f-793b-4895-98bf-fca131bc7f00","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a"}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a"}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:33.659Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:33.660Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:33.660Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7e4527de-9568-40aa-b675-3cae031e151a","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:34.668Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87"}
{"level":"info","ts":"2026-07-10T10:38:34.668Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87"}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","podName":"fail-rkvnj-head-p58jf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","podName":"fail-rkvnj-head-p58jf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:38:34.669Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"09eee718-bbc8-40f2-9fdd-3f1d9d771f87","name":"fail-rkvnj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"28cb59b2-5f8c-42c6-8ae7-2d48f047548a","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"dabc72e5-af4a-46c6-84f3-3ce1629e2a28","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cc8968b0-b5ab-4c40-9403-0e09ff7d2b90","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cc8968b0-b5ab-4c40-9403-0e09ff7d2b90","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cc8968b0-b5ab-4c40-9403-0e09ff7d2b90","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"dabc72e5-af4a-46c6-84f3-3ce1629e2a28","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"dabc72e5-af4a-46c6-84f3-3ce1629e2a28","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"dabc72e5-af4a-46c6-84f3-3ce1629e2a28","rayCluster":"fail-rkvnj","podIPs":["10.133.0.33","10.132.0.24"]}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cc8968b0-b5ab-4c40-9403-0e09ff7d2b90"}
{"level":"info","ts":"2026-07-10T10:38:34.684Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cc8968b0-b5ab-4c40-9403-0e09ff7d2b90","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:34.697Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"28cb59b2-5f8c-42c6-8ae7-2d48f047548a","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:34.708Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"28cb59b2-5f8c-42c6-8ae7-2d48f047548a","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:34.708Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"28cb59b2-5f8c-42c6-8ae7-2d48f047548a","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:34.751Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a1eba96-942f-4376-8c90-eb3a5337955b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:34.751Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a1eba96-942f-4376-8c90-eb3a5337955b","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:34.751Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a1eba96-942f-4376-8c90-eb3a5337955b","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:34.751Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a1eba96-942f-4376-8c90-eb3a5337955b"}
{"level":"info","ts":"2026-07-10T10:38:34.751Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0a1eba96-942f-4376-8c90-eb3a5337955b","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136"}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136"}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","podName":"fail-rkvnj-head-p58jf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","podName":"fail-rkvnj-head-p58jf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:36.684Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"e0b35609-246b-47d4-b244-6b7378e3f136","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:37.752Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fb920b52-a983-4084-8a39-7270fccc6694","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:37.752Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fb920b52-a983-4084-8a39-7270fccc6694","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:37.752Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fb920b52-a983-4084-8a39-7270fccc6694","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:37.752Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fb920b52-a983-4084-8a39-7270fccc6694"}
{"level":"info","ts":"2026-07-10T10:38:37.752Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fb920b52-a983-4084-8a39-7270fccc6694","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:40.753Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c05833c1-edc2-465d-ae3f-517b541f1f45","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:40.753Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c05833c1-edc2-465d-ae3f-517b541f1f45","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:40.753Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c05833c1-edc2-465d-ae3f-517b541f1f45","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:40.753Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c05833c1-edc2-465d-ae3f-517b541f1f45"}
{"level":"info","ts":"2026-07-10T10:38:40.753Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c05833c1-edc2-465d-ae3f-517b541f1f45","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:43.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"090fc089-d8d5-4242-8d21-84eb0d529f6a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:43.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"090fc089-d8d5-4242-8d21-84eb0d529f6a","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:43.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"090fc089-d8d5-4242-8d21-84eb0d529f6a","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:43.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"090fc089-d8d5-4242-8d21-84eb0d529f6a"}
{"level":"info","ts":"2026-07-10T10:38:43.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"090fc089-d8d5-4242-8d21-84eb0d529f6a","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:38:45.699Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"48b7debd-683d-4479-bef3-7f85dddda1f4","name":"fail-rkvnj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7d260fe4-c9a9-475e-9f92-4d044d888927","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7a44ec3b-2b26-4866-9e8d-034c3a441381","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"960c7ac9-4f12-4404-9f0d-cfb981f4e935","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"960c7ac9-4f12-4404-9f0d-cfb981f4e935","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"960c7ac9-4f12-4404-9f0d-cfb981f4e935","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7a44ec3b-2b26-4866-9e8d-034c3a441381","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7a44ec3b-2b26-4866-9e8d-034c3a441381","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7a44ec3b-2b26-4866-9e8d-034c3a441381","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"960c7ac9-4f12-4404-9f0d-cfb981f4e935"}
{"level":"info","ts":"2026-07-10T10:38:45.711Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"960c7ac9-4f12-4404-9f0d-cfb981f4e935","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:45.723Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7d260fe4-c9a9-475e-9f92-4d044d888927","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:38:45.734Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7d260fe4-c9a9-475e-9f92-4d044d888927","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:38:45.734Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7d260fe4-c9a9-475e-9f92-4d044d888927","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:46.755Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b56fb273-bcab-4c78-9bea-19c61389cd83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:46.755Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b56fb273-bcab-4c78-9bea-19c61389cd83","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:46.755Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b56fb273-bcab-4c78-9bea-19c61389cd83","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:46.755Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b56fb273-bcab-4c78-9bea-19c61389cd83"}
{"level":"info","ts":"2026-07-10T10:38:46.755Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b56fb273-bcab-4c78-9bea-19c61389cd83","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:38:47.712Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"4bb96bfa-450c-4bd9-b46c-e8889b612db8","seconds":300}
{"level":"info","ts":"2026-07-10T10:38:49.756Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"779cbf1f-b626-45b4-8c9f-25b726897dfc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:49.756Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"779cbf1f-b626-45b4-8c9f-25b726897dfc","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:49.756Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"779cbf1f-b626-45b4-8c9f-25b726897dfc","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:49.756Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"779cbf1f-b626-45b4-8c9f-25b726897dfc"}
{"level":"info","ts":"2026-07-10T10:38:49.756Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"779cbf1f-b626-45b4-8c9f-25b726897dfc","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:52.757Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"995e3243-7dc6-4ee6-8c28-7162ebcb811b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:52.757Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"995e3243-7dc6-4ee6-8c28-7162ebcb811b","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:52.757Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"995e3243-7dc6-4ee6-8c28-7162ebcb811b","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:52.757Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"995e3243-7dc6-4ee6-8c28-7162ebcb811b"}
{"level":"info","ts":"2026-07-10T10:38:52.757Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"995e3243-7dc6-4ee6-8c28-7162ebcb811b","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:55.758Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"dd3a1e69-5ccc-4c7d-b0cf-2a5614b4229c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:55.758Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"dd3a1e69-5ccc-4c7d-b0cf-2a5614b4229c","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:55.758Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"dd3a1e69-5ccc-4c7d-b0cf-2a5614b4229c","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:55.758Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"dd3a1e69-5ccc-4c7d-b0cf-2a5614b4229c"}
{"level":"info","ts":"2026-07-10T10:38:55.758Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"dd3a1e69-5ccc-4c7d-b0cf-2a5614b4229c","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:38:56.251Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f68077be-bd77-4f6e-ae5b-644f65e7bf8f","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:56.251Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f68077be-bd77-4f6e-ae5b-644f65e7bf8f","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:38:56.251Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f68077be-bd77-4f6e-ae5b-644f65e7bf8f","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:38:56.251Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f68077be-bd77-4f6e-ae5b-644f65e7bf8f","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:38:58.759Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1540bf20-8152-4e3e-ba9c-3c73c8d16607","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:38:58.759Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1540bf20-8152-4e3e-ba9c-3c73c8d16607","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:38:58.759Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1540bf20-8152-4e3e-ba9c-3c73c8d16607","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:38:58.760Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1540bf20-8152-4e3e-ba9c-3c73c8d16607"}
{"level":"info","ts":"2026-07-10T10:38:58.760Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1540bf20-8152-4e3e-ba9c-3c73c8d16607","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:01.760Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c52fd90e-b6e9-408a-971f-c727445b7fee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:01.760Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c52fd90e-b6e9-408a-971f-c727445b7fee","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:01.761Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c52fd90e-b6e9-408a-971f-c727445b7fee","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:01.761Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c52fd90e-b6e9-408a-971f-c727445b7fee"}
{"level":"info","ts":"2026-07-10T10:39:01.761Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c52fd90e-b6e9-408a-971f-c727445b7fee","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:04.761Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"3cb0fb76-d16f-4043-8412-34fcc49dec96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:04.761Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"3cb0fb76-d16f-4043-8412-34fcc49dec96","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:04.761Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"3cb0fb76-d16f-4043-8412-34fcc49dec96","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:04.761Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"3cb0fb76-d16f-4043-8412-34fcc49dec96"}
{"level":"info","ts":"2026-07-10T10:39:04.761Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"3cb0fb76-d16f-4043-8412-34fcc49dec96","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:07.762Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"af4aa08e-6f71-49b7-aa0f-08981b61d946","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:07.762Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"af4aa08e-6f71-49b7-aa0f-08981b61d946","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:07.762Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"af4aa08e-6f71-49b7-aa0f-08981b61d946","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:07.762Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"af4aa08e-6f71-49b7-aa0f-08981b61d946"}
{"level":"info","ts":"2026-07-10T10:39:07.762Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"af4aa08e-6f71-49b7-aa0f-08981b61d946","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:10.359Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8ac50e46-ebc4-4ad1-b406-9785625efe45","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:39:10.359Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8ac50e46-ebc4-4ad1-b406-9785625efe45","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:39:10.359Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8ac50e46-ebc4-4ad1-b406-9785625efe45","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:39:10.359Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8ac50e46-ebc4-4ad1-b406-9785625efe45","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:39:10.762Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a2103d00-a9ac-40ca-baed-46d49d119fb7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:10.762Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a2103d00-a9ac-40ca-baed-46d49d119fb7","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:10.762Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a2103d00-a9ac-40ca-baed-46d49d119fb7","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:10.763Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a2103d00-a9ac-40ca-baed-46d49d119fb7"}
{"level":"info","ts":"2026-07-10T10:39:10.763Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"a2103d00-a9ac-40ca-baed-46d49d119fb7","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:13.764Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"791b4bc4-0e04-4fbe-aa8f-232795a85126","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:13.764Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"791b4bc4-0e04-4fbe-aa8f-232795a85126","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:13.764Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"791b4bc4-0e04-4fbe-aa8f-232795a85126","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:13.764Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"791b4bc4-0e04-4fbe-aa8f-232795a85126"}
{"level":"info","ts":"2026-07-10T10:39:13.764Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"791b4bc4-0e04-4fbe-aa8f-232795a85126","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:16.765Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9289d29f-0bb6-4c10-aa91-d0819b508902","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:16.765Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9289d29f-0bb6-4c10-aa91-d0819b508902","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:16.765Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9289d29f-0bb6-4c10-aa91-d0819b508902","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:16.765Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9289d29f-0bb6-4c10-aa91-d0819b508902"}
{"level":"info","ts":"2026-07-10T10:39:16.765Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"9289d29f-0bb6-4c10-aa91-d0819b508902","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:19.766Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7130f361-daa5-42a2-b280-b1f4d6e01625","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:19.766Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7130f361-daa5-42a2-b280-b1f4d6e01625","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:19.766Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7130f361-daa5-42a2-b280-b1f4d6e01625","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:19.766Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7130f361-daa5-42a2-b280-b1f4d6e01625"}
{"level":"info","ts":"2026-07-10T10:39:19.766Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7130f361-daa5-42a2-b280-b1f4d6e01625","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:22.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"81a53b40-e05e-4e5d-82f9-b831ead5e099","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:22.767Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"81a53b40-e05e-4e5d-82f9-b831ead5e099","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:22.767Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"81a53b40-e05e-4e5d-82f9-b831ead5e099","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:22.767Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"81a53b40-e05e-4e5d-82f9-b831ead5e099"}
{"level":"info","ts":"2026-07-10T10:39:22.767Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"81a53b40-e05e-4e5d-82f9-b831ead5e099","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:25.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cdcbab9b-825a-4517-8f3e-d8dfd187a6f8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:25.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cdcbab9b-825a-4517-8f3e-d8dfd187a6f8","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:25.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cdcbab9b-825a-4517-8f3e-d8dfd187a6f8","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:25.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cdcbab9b-825a-4517-8f3e-d8dfd187a6f8"}
{"level":"info","ts":"2026-07-10T10:39:25.768Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"cdcbab9b-825a-4517-8f3e-d8dfd187a6f8","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:28.769Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b216789a-4eca-4294-afac-c73dce21144a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:28.769Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b216789a-4eca-4294-afac-c73dce21144a","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:28.769Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b216789a-4eca-4294-afac-c73dce21144a","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:28.770Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b216789a-4eca-4294-afac-c73dce21144a"}
{"level":"info","ts":"2026-07-10T10:39:28.770Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"b216789a-4eca-4294-afac-c73dce21144a","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:31.770Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"bdb64e53-64a4-41f8-b875-349baa1cc41d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:31.770Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"bdb64e53-64a4-41f8-b875-349baa1cc41d","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:31.770Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"bdb64e53-64a4-41f8-b875-349baa1cc41d","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:31.770Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"bdb64e53-64a4-41f8-b875-349baa1cc41d"}
{"level":"info","ts":"2026-07-10T10:39:31.770Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"bdb64e53-64a4-41f8-b875-349baa1cc41d","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b"}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b"}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:39:33.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:33.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:39:33.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:39:33.375Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:39:33.375Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"67f8cae6-11f2-43e4-b765-071f2b8e179b","seconds":300}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:39:34.363Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"eea051a9-1fca-4662-ad8f-5c34727d23b7","seconds":300}
{"level":"info","ts":"2026-07-10T10:39:34.771Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7e6d7403-4c9f-4a2c-bf26-135b5e23ded4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:34.771Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7e6d7403-4c9f-4a2c-bf26-135b5e23ded4","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:34.771Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7e6d7403-4c9f-4a2c-bf26-135b5e23ded4","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:34.772Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7e6d7403-4c9f-4a2c-bf26-135b5e23ded4"}
{"level":"info","ts":"2026-07-10T10:39:34.772Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"7e6d7403-4c9f-4a2c-bf26-135b5e23ded4","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:37.772Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"97c5fa1e-4796-4fa8-8ca7-5c26a34f6167","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:37.772Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"97c5fa1e-4796-4fa8-8ca7-5c26a34f6167","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:37.772Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"97c5fa1e-4796-4fa8-8ca7-5c26a34f6167","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:37.772Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"97c5fa1e-4796-4fa8-8ca7-5c26a34f6167"}
{"level":"info","ts":"2026-07-10T10:39:37.772Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"97c5fa1e-4796-4fa8-8ca7-5c26a34f6167","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:40.773Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"616001b3-d1f1-4760-b555-6b856ea6cc94","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:40.774Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"616001b3-d1f1-4760-b555-6b856ea6cc94","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:40.774Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"616001b3-d1f1-4760-b555-6b856ea6cc94","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:40.774Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"616001b3-d1f1-4760-b555-6b856ea6cc94"}
{"level":"info","ts":"2026-07-10T10:39:40.774Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"616001b3-d1f1-4760-b555-6b856ea6cc94","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:39:43.404Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6e6bfb88-2382-4c5e-9e1a-6ae5dd798d69","seconds":300}
{"level":"info","ts":"2026-07-10T10:39:43.775Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"528c1f94-2e8a-4dfe-87d5-da1300e48d8c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:43.775Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"528c1f94-2e8a-4dfe-87d5-da1300e48d8c","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:43.775Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"528c1f94-2e8a-4dfe-87d5-da1300e48d8c","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:43.775Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"528c1f94-2e8a-4dfe-87d5-da1300e48d8c"}
{"level":"info","ts":"2026-07-10T10:39:43.775Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"528c1f94-2e8a-4dfe-87d5-da1300e48d8c","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49"}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49"}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:39:44.409Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"88cadb91-8763-451b-afb2-aab67bc62e49","name":"fail-rkvnj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:38:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7456d1db-1c1f-440f-ae98-506328b6e4c9","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4a972374-092a-4d4e-9dec-4affeff79cae","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fd87dace-983e-46b0-9373-2cc673aa4f66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fd87dace-983e-46b0-9373-2cc673aa4f66","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fd87dace-983e-46b0-9373-2cc673aa4f66","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7456d1db-1c1f-440f-ae98-506328b6e4c9","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7456d1db-1c1f-440f-ae98-506328b6e4c9","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fd87dace-983e-46b0-9373-2cc673aa4f66"}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"fd87dace-983e-46b0-9373-2cc673aa4f66","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:44.422Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7456d1db-1c1f-440f-ae98-506328b6e4c9","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:39:44.438Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4a972374-092a-4d4e-9dec-4affeff79cae","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:39:44.448Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4a972374-092a-4d4e-9dec-4affeff79cae","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:39:44.448Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"4a972374-092a-4d4e-9dec-4affeff79cae","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:46.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454"}
{"level":"info","ts":"2026-07-10T10:39:46.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454"}
{"level":"info","ts":"2026-07-10T10:39:46.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:39:46.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:39:46.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:46.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:38:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:39:46.422Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ebeeeb6b-5251-4e86-94bb-b091f6e20454","seconds":300}
{"level":"info","ts":"2026-07-10T10:39:46.775Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"e3b9c26a-0062-4bf1-b4ff-d12a5819f668","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:46.775Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"e3b9c26a-0062-4bf1-b4ff-d12a5819f668","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:46.775Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"e3b9c26a-0062-4bf1-b4ff-d12a5819f668","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:46.776Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"e3b9c26a-0062-4bf1-b4ff-d12a5819f668"}
{"level":"info","ts":"2026-07-10T10:39:46.776Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"e3b9c26a-0062-4bf1-b4ff-d12a5819f668","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:49.776Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"62872ce5-e9c6-4be3-8c6a-58d2c3d06de9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:49.776Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"62872ce5-e9c6-4be3-8c6a-58d2c3d06de9","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:49.776Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"62872ce5-e9c6-4be3-8c6a-58d2c3d06de9","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:49.777Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"62872ce5-e9c6-4be3-8c6a-58d2c3d06de9"}
{"level":"info","ts":"2026-07-10T10:39:49.777Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"62872ce5-e9c6-4be3-8c6a-58d2c3d06de9","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:52.778Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6a2f1a3a-aee4-4953-9de8-f6f024214aba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:52.778Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6a2f1a3a-aee4-4953-9de8-f6f024214aba","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:52.778Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6a2f1a3a-aee4-4953-9de8-f6f024214aba","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:52.778Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6a2f1a3a-aee4-4953-9de8-f6f024214aba"}
{"level":"info","ts":"2026-07-10T10:39:52.778Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6a2f1a3a-aee4-4953-9de8-f6f024214aba","RayCluster":"fail-rkvnj","State":""}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7"}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7"}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:39:55.432Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"c270c642-3785-4dde-96ca-17c95fa7e9c7","name":"fail-rkvnj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:55Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:39:55.445Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"391e8b47-f0b1-4d30-9db2-0cea44576c6b","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:55.445Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"bb3976b0-50af-4bc1-83df-92b26781ce8c","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"bb3976b0-50af-4bc1-83df-92b26781ce8c","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"bb3976b0-50af-4bc1-83df-92b26781ce8c","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"bb3976b0-50af-4bc1-83df-92b26781ce8c","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","head service name":"fail-rkvnj-head-svc","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","head service URL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","SubmissionMode":"HTTPMode","RayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","dashboardURL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:39:55.446Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"32095095-d5d2-46aa-bcd6-64acc4f594c0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:39:55.455Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"101f5b6d-ffbd-4225-bfa0-64780736d869","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:55.455Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"101f5b6d-ffbd-4225-bfa0-64780736d869","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:55.455Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"101f5b6d-ffbd-4225-bfa0-64780736d869","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:55.455Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"101f5b6d-ffbd-4225-bfa0-64780736d869"}
{"level":"info","ts":"2026-07-10T10:39:55.463Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"391e8b47-f0b1-4d30-9db2-0cea44576c6b","name":"fail-rkvnj-head"}
{"level":"info","ts":"2026-07-10T10:39:55.465Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"101f5b6d-ffbd-4225-bfa0-64780736d869","JobId":"fail-cgvsd"}
{"level":"info","ts":"2026-07-10T10:39:55.465Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"101f5b6d-ffbd-4225-bfa0-64780736d869","rayJob":"fail","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/fail.py\",\"submission_id\":\"fail-cgvsd\"}"}
{"level":"info","ts":"2026-07-10T10:39:55.472Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"391e8b47-f0b1-4d30-9db2-0cea44576c6b","name":"fail-rkvnj-workers"}
{"level":"info","ts":"2026-07-10T10:39:55.472Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"391e8b47-f0b1-4d30-9db2-0cea44576c6b","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:56.251Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"d0608e2b-7104-407e-9466-d764db7cd04a","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:56.252Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"d0608e2b-7104-407e-9466-d764db7cd04a","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:39:56.252Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"d0608e2b-7104-407e-9466-d764db7cd04a","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:39:56.252Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"d0608e2b-7104-407e-9466-d764db7cd04a","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:39:56.370Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6f6ea099-4c36-43bf-9fd9-f0c07345318c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:56.370Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6f6ea099-4c36-43bf-9fd9-f0c07345318c","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:56.370Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6f6ea099-4c36-43bf-9fd9-f0c07345318c","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:56.371Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6f6ea099-4c36-43bf-9fd9-f0c07345318c"}
{"level":"info","ts":"2026-07-10T10:39:56.373Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6f6ea099-4c36-43bf-9fd9-f0c07345318c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","dashboardURL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:39:56Z"},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","dashboardURL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:55Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:39:56.374Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6f6ea099-4c36-43bf-9fd9-f0c07345318c","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:39:56.387Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"d11fef67-f5ae-4874-bac7-fbee15d614e0","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:56.387Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"d11fef67-f5ae-4874-bac7-fbee15d614e0","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:56.387Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"d11fef67-f5ae-4874-bac7-fbee15d614e0","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:56.387Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"d11fef67-f5ae-4874-bac7-fbee15d614e0"}
{"level":"info","ts":"2026-07-10T10:39:56.390Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"d11fef67-f5ae-4874-bac7-fbee15d614e0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:39:56Z"},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","dashboardURL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:55Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:39:56Z"},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","dashboardURL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:55Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:39:57.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787"}
{"level":"info","ts":"2026-07-10T10:39:57.445Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787"}
{"level":"info","ts":"2026-07-10T10:39:57.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:39:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:39:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:39:57.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:39:57.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:57.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:39:57.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:39:57.446Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:39:57.446Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"80800ec5-e004-4422-92f3-3446fbe07787","seconds":300}
{"level":"info","ts":"2026-07-10T10:39:59.371Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"5dd5839e-9360-41a8-8f30-0eb3ecf525e0","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:59.371Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"5dd5839e-9360-41a8-8f30-0eb3ecf525e0","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:39:59.371Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"5dd5839e-9360-41a8-8f30-0eb3ecf525e0","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:39:59.371Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"5dd5839e-9360-41a8-8f30-0eb3ecf525e0"}
{"level":"info","ts":"2026-07-10T10:39:59.374Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"5dd5839e-9360-41a8-8f30-0eb3ecf525e0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:39:56Z"},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","dashboardURL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:55Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:39:56Z","endTime":"2026-07-10T10:39:57Z"},"jobId":"fail-cgvsd","rayClusterName":"fail-rkvnj","dashboardURL":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:39:56,041\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-cgvsd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:38:25Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:55Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:39:59.375Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"5dd5839e-9360-41a8-8f30-0eb3ecf525e0","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-10T10:39:59.389Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"60ae5bfe-19de-4736-aaab-f2ffbff419ca","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:39:59.389Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"60ae5bfe-19de-4736-aaab-f2ffbff419ca","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:39:59 +0000 UTC","Now":"2026-07-10T10:39:59.389Z","ShutdownTime":"2026-07-10T10:39:59.000Z"}
{"level":"info","ts":"2026-07-10T10:40:00.289Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a6301315-7546-45d4-b2a7-f9456bcbc124","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:00.289Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a6301315-7546-45d4-b2a7-f9456bcbc124","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:40:00.299Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a6301315-7546-45d4-b2a7-f9456bcbc124"}
{"level":"info","ts":"2026-07-10T10:40:00.299Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a6301315-7546-45d4-b2a7-f9456bcbc124","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:40:00.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a6301315-7546-45d4-b2a7-f9456bcbc124","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:40:00.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a6301315-7546-45d4-b2a7-f9456bcbc124","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:40:00.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7b082891-e464-45e0-9060-b7739beccd07","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:00.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7b082891-e464-45e0-9060-b7739beccd07","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.309Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7b082891-e464-45e0-9060-b7739beccd07","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.309Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7b082891-e464-45e0-9060-b7739beccd07","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.325Z","logger":"raycluster-resource","msg":"default","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.325Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-klvsc","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7b082891-e464-45e0-9060-b7739beccd07","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bed0d23d-022a-4992-881f-a9e9d361c527","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7b082891-e464-45e0-9060-b7739beccd07"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","mode":"IntegratedOAuth","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7b082891-e464-45e0-9060-b7739beccd07","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"0eb80a58-ef39-4729-a234-48baf8baf620","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"0eb80a58-ef39-4729-a234-48baf8baf620","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"0eb80a58-ef39-4729-a234-48baf8baf620","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.335Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.336Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"0eb80a58-ef39-4729-a234-48baf8baf620"}
{"level":"info","ts":"2026-07-10T10:40:00.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"0eb80a58-ef39-4729-a234-48baf8baf620","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:00.340Z","logger":"raycluster-resource","msg":"default","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.341Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-klvsc","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","name":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:40:00.344Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","name":"stop-klvsc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:00.345Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.345Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.350Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59c6eb7b-cc04-49ef-870e-63b9863f1f0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:00.350Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59c6eb7b-cc04-49ef-870e-63b9863f1f0b","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.350Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59c6eb7b-cc04-49ef-870e-63b9863f1f0b","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59c6eb7b-cc04-49ef-870e-63b9863f1f0b"}
{"level":"info","ts":"2026-07-10T10:40:00.350Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59c6eb7b-cc04-49ef-870e-63b9863f1f0b","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bed0d23d-022a-4992-881f-a9e9d361c527","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","name":"stop-klvsc-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","name":"stop-klvsc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-klvsc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"stop-klvsc"}}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c9c38f3-63f1-4bde-b817-99275a1d9f88","cluster":"stop-klvsc"}
{"level":"error","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"40a8bcad-7695-4e33-ae93-050a2df79041","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:00.425Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.426Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:40:00.426Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","name":"stop-klvsc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:00.525Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.526Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"56ed1800-c891-4100-8578-30e5939617bd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:00.526Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"56ed1800-c891-4100-8578-30e5939617bd","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.526Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"56ed1800-c891-4100-8578-30e5939617bd","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.526Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"56ed1800-c891-4100-8578-30e5939617bd"}
{"level":"info","ts":"2026-07-10T10:40:00.526Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"56ed1800-c891-4100-8578-30e5939617bd","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:00.625Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","name":"test-ns-7lp9t-stop-klvsc","operation":"created"}
{"level":"info","ts":"2026-07-10T10:40:00.625Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a"}
{"level":"error","ts":"2026-07-10T10:40:00.625Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a47163d4-2ec6-46dc-a195-c5318a483f2a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d"}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d"}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d"}
{"level":"error","ts":"2026-07-10T10:40:00.626Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"94b78245-8263-42a5-a1f1-34824946183d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.631Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","name":"kube-rbac-proxy-config-stop-klvsc","operation":"created"}
{"level":"info","ts":"2026-07-10T10:40:00.631Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c9c38f3-63f1-4bde-b817-99275a1d9f88","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d"}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d"}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d"}
{"level":"error","ts":"2026-07-10T10:40:00.636Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"89c77a1c-1139-4e6c-949e-4b2eeb06a05d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.725Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:40:00.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2"}
{"level":"info","ts":"2026-07-10T10:40:00.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2"}
{"level":"info","ts":"2026-07-10T10:40:00.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:00.725Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.725Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.725Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-07-10T10:40:00.726Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-klvsc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938"}
{"level":"error","ts":"2026-07-10T10:40:00.726Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2f3e6012-2c24-4bb5-a584-cf177754a938","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-klvsc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2"}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c052f808-7038-4ef1-819a-afa2d3d1a55e","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"error","ts":"2026-07-10T10:40:00.726Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a52ef2ff-7ac8-4aee-bf53-1bf4f5141af2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c052f808-7038-4ef1-819a-afa2d3d1a55e","mode":"IntegratedOAuth","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.726Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c052f808-7038-4ef1-819a-afa2d3d1a55e","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.730Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c9c38f3-63f1-4bde-b817-99275a1d9f88","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:00.730Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c9c38f3-63f1-4bde-b817-99275a1d9f88","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.730Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"a7402086-469e-4d30-8ceb-b1f77b78667e","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.736Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d","rayCluster":"stop-klvsc","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:40:00.736Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bc03e48b-e4d0-4e45-bca4-b5089667d74d"}
{"level":"info","ts":"2026-07-10T10:40:00.736Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"263f9a1d-be9c-4741-ba43-2d07dfa7320c","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.736Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"263f9a1d-be9c-4741-ba43-2d07dfa7320c"}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c052f808-7038-4ef1-819a-afa2d3d1a55e","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"98c4c3fe-f2e5-4598-9d8f-9d87a4af9aa9","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"98c4c3fe-f2e5-4598-9d8f-9d87a4af9aa9","mode":"IntegratedOAuth","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"98c4c3fe-f2e5-4598-9d8f-9d87a4af9aa9","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"695b17d7-6e7d-48c0-a740-f235f930acee","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"b811f387-32b3-420f-8ec6-6b2f146eaf41","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"b811f387-32b3-420f-8ec6-6b2f146eaf41","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"b811f387-32b3-420f-8ec6-6b2f146eaf41","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.742Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"695b17d7-6e7d-48c0-a740-f235f930acee"}
{"level":"info","ts":"2026-07-10T10:40:00.746Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"a7402086-469e-4d30-8ceb-b1f77b78667e","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"b811f387-32b3-420f-8ec6-6b2f146eaf41"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"b811f387-32b3-420f-8ec6-6b2f146eaf41","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.825Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"98c4c3fe-f2e5-4598-9d8f-9d87a4af9aa9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-klvsc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"98c4c3fe-f2e5-4598-9d8f-9d87a4af9aa9"}
{"level":"error","ts":"2026-07-10T10:40:00.825Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"98c4c3fe-f2e5-4598-9d8f-9d87a4af9aa9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-klvsc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f"}
{"level":"error","ts":"2026-07-10T10:40:00.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"fa734954-af79-40b8-b123-bf6040e2b81f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.829Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"a7402086-469e-4d30-8ceb-b1f77b78667e","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:00.829Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"a7402086-469e-4d30-8ceb-b1f77b78667e","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.829Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7a341b98-dce2-4493-8066-9dc5aeff4dff","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.831Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"81a8af47-2c81-4918-8949-c2076b0e954b","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:00.831Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"81a8af47-2c81-4918-8949-c2076b0e954b","mode":"IntegratedOAuth","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.831Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"81a8af47-2c81-4918-8949-c2076b0e954b","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.841Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"81a8af47-2c81-4918-8949-c2076b0e954b","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.842Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7a341b98-dce2-4493-8066-9dc5aeff4dff","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:00.852Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7a341b98-dce2-4493-8066-9dc5aeff4dff","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:00.852Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7a341b98-dce2-4493-8066-9dc5aeff4dff","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f"}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f"}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f"}
{"level":"error","ts":"2026-07-10T10:40:00.986Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"61b3f57d-dabc-4e2b-ab90-0d53ce8dad0f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe"}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe"}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:01.306Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe"}
{"level":"error","ts":"2026-07-10T10:40:01.307Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"14a8431d-d72f-47e1-8b27-a505baeb8fbe","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a"}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a"}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","error":"head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"error","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found"}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"stop-klvsc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a"}
{"level":"error","ts":"2026-07-10T10:40:01.948Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e09b6ece-7a37-45e6-84c3-6b6c98b9484a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-klvsc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:02.389Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1b027bb4-ed5d-46ae-82f9-966bb90d9697","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:02.389Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"1b027bb4-ed5d-46ae-82f9-966bb90d9697","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:39:59 +0000 UTC","Now":"2026-07-10T10:40:02.389Z","ShutdownTime":"2026-07-10T10:39:59.000Z"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","mode":"IntegratedOAuth","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","cluster":"stop-klvsc","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","cluster":"stop-klvsc","authType":"IntegratedOAuth","serviceAccount":"stop-klvsc-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-klvsc","ray.io/group":"headgroup","ray.io/identifier":"stop-klvsc-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:40:03.228Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","enabled":true}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","name":"stop-klvsc-head-m2plk"}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab"}
{"level":"info","ts":"2026-07-10T10:40:03.257Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","nodeType":"worker","rayStartParams":{"address":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:40:03.258Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","rayStartCmd":"ray start  --address=stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:40:03.258Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:40:03.258Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","enabled":true}
{"level":"info","ts":"2026-07-10T10:40:03.275Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","name":"stop-klvsc-small-group-worker-tzdvk"}
{"level":"info","ts":"2026-07-10T10:40:03.275Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.275Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-klvsc-head-svc"},"newHead":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"}}
{"level":"info","ts":"2026-07-10T10:40:03.275Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"df0376dc-b8a2-429c-8b41-cb86f071c6ab","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:03.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7"}
{"level":"info","ts":"2026-07-10T10:40:03.289Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7"}
{"level":"info","ts":"2026-07-10T10:40:03.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:03.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-klvsc-head-svc"},"newHead":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"}}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"6ec27d10-5500-442f-a4b7-4a002f8d0ca6","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"5c389ad6-7067-4baa-a500-db4d4a033a6f","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7e5485a1-3514-4c43-9972-134a86e0594f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7e5485a1-3514-4c43-9972-134a86e0594f","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7e5485a1-3514-4c43-9972-134a86e0594f","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"5c389ad6-7067-4baa-a500-db4d4a033a6f","rayCluster":"stop-klvsc","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7e5485a1-3514-4c43-9972-134a86e0594f"}
{"level":"info","ts":"2026-07-10T10:40:03.290Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7e5485a1-3514-4c43-9972-134a86e0594f","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:03.297Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","name":"stop-klvsc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-klvsc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"stop-klvsc"}}
{"level":"info","ts":"2026-07-10T10:40:03.297Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7"}
{"level":"error","ts":"2026-07-10T10:40:03.297Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"c9ed3047-1333-480a-a0b5-a3266fd3c1c7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-klvsc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"6ec27d10-5500-442f-a4b7-4a002f8d0ca6","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"740b539c-9ada-4bcc-a014-c23237d05bc4","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.303Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:03.304Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a455d4cd-6601-46c1-840e-0c3c14efe737","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"616f0763-9230-469d-849c-cb4ea9be8c6a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"616f0763-9230-469d-849c-cb4ea9be8c6a","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"616f0763-9230-469d-849c-cb4ea9be8c6a","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"616f0763-9230-469d-849c-cb4ea9be8c6a"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"616f0763-9230-469d-849c-cb4ea9be8c6a","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:40:03.325Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"da29e663-3a2f-4f83-96e9-e6edeeb0e887","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:03.335Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"6ec27d10-5500-442f-a4b7-4a002f8d0ca6","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:03.335Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"6ec27d10-5500-442f-a4b7-4a002f8d0ca6","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"647bf96a-b015-4c7b-be43-f767638927a2","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a256dc78-6cef-4c7d-832c-399bfc70ce31","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a256dc78-6cef-4c7d-832c-399bfc70ce31","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a256dc78-6cef-4c7d-832c-399bfc70ce31","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9f95d2ed-75bf-46b5-8c91-f83002275e32","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a256dc78-6cef-4c7d-832c-399bfc70ce31"}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a256dc78-6cef-4c7d-832c-399bfc70ce31","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:03.345Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9f95d2ed-75bf-46b5-8c91-f83002275e32","rayCluster":"stop-klvsc","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:03.346Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3b472e21-bd32-4cfc-a28a-9e76f78376d5","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:03.357Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"647bf96a-b015-4c7b-be43-f767638927a2","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:03.429Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"647bf96a-b015-4c7b-be43-f767638927a2","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:03.429Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"647bf96a-b015-4c7b-be43-f767638927a2","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:03.709Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae91f61a-fd5c-4b67-8e7e-8a3f927dbcf1","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:03.739Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"42f08450-aa4b-4686-abbd-8e28cfe60c19","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:04.022Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9c5fec93-1d93-48ea-b627-dde65e3cd151","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"newHead":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"}}
{"level":"info","ts":"2026-07-10T10:40:04.496Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"229a5ef1-0eb6-4af7-9c3b-0b82495ca4c2","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"28317370-1e91-4b74-8d03-0b539b68bac1","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"41919d23-c3c0-4686-9873-861db157b5b3","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ecaea50a-42ec-405f-90c5-8ec40f69da59","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"41919d23-c3c0-4686-9873-861db157b5b3","pod":"stop-klvsc-head-m2plk","ip":"10.133.0.34"}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ecaea50a-42ec-405f-90c5-8ec40f69da59","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"41919d23-c3c0-4686-9873-861db157b5b3","pod":"stop-klvsc-small-group-worker-tzdvk","ip":"10.132.0.25"}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ecaea50a-42ec-405f-90c5-8ec40f69da59","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ecaea50a-42ec-405f-90c5-8ec40f69da59"}
{"level":"info","ts":"2026-07-10T10:40:04.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ecaea50a-42ec-405f-90c5-8ec40f69da59","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:04.519Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"28317370-1e91-4b74-8d03-0b539b68bac1","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:04.519Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"41919d23-c3c0-4686-9873-861db157b5b3","certificate":"ray-head-cert-stop-klvsc","dnsNames":["stop-klvsc-head-svc","localhost","stop-klvsc-head-svc.test-ns-7lp9t.svc","stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.133.0.34","10.132.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:40:04.531Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"28317370-1e91-4b74-8d03-0b539b68bac1","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:04.531Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"28317370-1e91-4b74-8d03-0b539b68bac1","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:04.532Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"41919d23-c3c0-4686-9873-861db157b5b3","certificate":"ray-worker-cert-stop-klvsc","dnsNames":["stop-klvsc-worker-svc","localhost","stop-klvsc-small-group","stop-klvsc-worker-svc.test-ns-7lp9t.svc","stop-klvsc-worker-svc.test-ns-7lp9t.svc.cluster.local","*.stop-klvsc-worker-svc.test-ns-7lp9t.svc","*.stop-klvsc-worker-svc.test-ns-7lp9t.svc.cluster.local","*-worker-*.test-ns-7lp9t.svc","*-worker-*.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.133.0.34","10.132.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:40:04.532Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"41919d23-c3c0-4686-9873-861db157b5b3","rayCluster":"stop-klvsc","podIPs":["10.133.0.34","10.132.0.25"]}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:05.346Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"362da1cb-7aef-4227-8415-11ca4cb73250","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:06.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c0150a49-124f-42fa-a250-72f4ba834b68","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:06.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c0150a49-124f-42fa-a250-72f4ba834b68","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:06.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c0150a49-124f-42fa-a250-72f4ba834b68","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:06.326Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c0150a49-124f-42fa-a250-72f4ba834b68"}
{"level":"info","ts":"2026-07-10T10:40:06.326Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c0150a49-124f-42fa-a250-72f4ba834b68","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:09.044Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"ae395406-f508-470d-af89-88b348518a46","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:09.327Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7c990f16-d071-497a-8962-0cedc42efeb9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:09.327Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7c990f16-d071-497a-8962-0cedc42efeb9","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:09.327Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7c990f16-d071-497a-8962-0cedc42efeb9","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:09.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7c990f16-d071-497a-8962-0cedc42efeb9"}
{"level":"info","ts":"2026-07-10T10:40:09.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7c990f16-d071-497a-8962-0cedc42efeb9","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:09.517Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621"}
{"level":"info","ts":"2026-07-10T10:40:09.517Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621"}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:09.518Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"e7b07ad4-dd0e-47a9-b4c9-2ad39de59621","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","podName":"stop-klvsc-head-m2plk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:10.055Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"30634ab2-8a46-4275-aba8-5e8bc9e26467","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:10.359Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"505c4e1a-f7b6-4842-b5b0-6d3c3cf33f12","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:40:10.359Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"505c4e1a-f7b6-4842-b5b0-6d3c3cf33f12","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:40:10.359Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"505c4e1a-f7b6-4842-b5b0-6d3c3cf33f12","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:40:10.359Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"505c4e1a-f7b6-4842-b5b0-6d3c3cf33f12","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:40:12.328Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"06c020da-0608-43d2-93cd-0ad8f1c46ba3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:12.328Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"06c020da-0608-43d2-93cd-0ad8f1c46ba3","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:12.328Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"06c020da-0608-43d2-93cd-0ad8f1c46ba3","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:12.328Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"06c020da-0608-43d2-93cd-0ad8f1c46ba3"}
{"level":"info","ts":"2026-07-10T10:40:12.328Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"06c020da-0608-43d2-93cd-0ad8f1c46ba3","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:12.539Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58"}
{"level":"info","ts":"2026-07-10T10:40:12.539Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58"}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:40:12.540Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"a9e0c4ea-4252-47ca-b260-97f4e5b67f58","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bccc27b2-0648-46b8-aa31-113aa127f1dd","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7260262a-9884-47dd-9cd9-a7c58fdacba9","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"6e27c353-752a-4beb-ade7-464e4f13ef71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7260262a-9884-47dd-9cd9-a7c58fdacba9","pod":"stop-klvsc-head-m2plk","ip":"10.133.0.34"}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"6e27c353-752a-4beb-ade7-464e4f13ef71","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"6e27c353-752a-4beb-ade7-464e4f13ef71","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7260262a-9884-47dd-9cd9-a7c58fdacba9","pod":"stop-klvsc-small-group-worker-tzdvk","ip":"10.132.0.25"}
{"level":"info","ts":"2026-07-10T10:40:12.552Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"7260262a-9884-47dd-9cd9-a7c58fdacba9","rayCluster":"stop-klvsc","podIPs":["10.133.0.34","10.132.0.25"]}
{"level":"info","ts":"2026-07-10T10:40:12.553Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"6e27c353-752a-4beb-ade7-464e4f13ef71"}
{"level":"info","ts":"2026-07-10T10:40:12.553Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"6e27c353-752a-4beb-ade7-464e4f13ef71","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:12.563Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bccc27b2-0648-46b8-aa31-113aa127f1dd","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:12.576Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bccc27b2-0648-46b8-aa31-113aa127f1dd","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:12.576Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bccc27b2-0648-46b8-aa31-113aa127f1dd","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:14.552Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"092838ea-90bb-497b-9b08-45c7e1605354","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:15.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7189f861-4d37-43a6-931f-d1af68998ef9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:15.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7189f861-4d37-43a6-931f-d1af68998ef9","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:15.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7189f861-4d37-43a6-931f-d1af68998ef9","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:15.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7189f861-4d37-43a6-931f-d1af68998ef9"}
{"level":"info","ts":"2026-07-10T10:40:15.329Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7189f861-4d37-43a6-931f-d1af68998ef9","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:18.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"93648b62-5bc7-44cb-8bea-9561318f11a7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:18.331Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"93648b62-5bc7-44cb-8bea-9561318f11a7","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:18.331Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"93648b62-5bc7-44cb-8bea-9561318f11a7","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:18.331Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"93648b62-5bc7-44cb-8bea-9561318f11a7"}
{"level":"info","ts":"2026-07-10T10:40:18.331Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"93648b62-5bc7-44cb-8bea-9561318f11a7","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:20.110Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"9e595c12-c79e-468a-960b-28abd9c39547","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f"}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f"}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:40:21.106Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"08d3668b-63a6-4a1f-a1ba-8f825384388f","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"94dc2946-5266-4c2d-bb12-70d511485d61","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c3c3de10-67dc-4313-8618-7d399c5ced23","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"490f58df-f393-490f-a1eb-8ec40bb112c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"490f58df-f393-490f-a1eb-8ec40bb112c1","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"490f58df-f393-490f-a1eb-8ec40bb112c1","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c3c3de10-67dc-4313-8618-7d399c5ced23","pod":"stop-klvsc-head-m2plk","ip":"10.133.0.34"}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c3c3de10-67dc-4313-8618-7d399c5ced23","pod":"stop-klvsc-small-group-worker-tzdvk","ip":"10.132.0.25"}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c3c3de10-67dc-4313-8618-7d399c5ced23","rayCluster":"stop-klvsc","podIPs":["10.133.0.34","10.132.0.25"]}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"490f58df-f393-490f-a1eb-8ec40bb112c1"}
{"level":"info","ts":"2026-07-10T10:40:21.119Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"490f58df-f393-490f-a1eb-8ec40bb112c1","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:21.132Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"94dc2946-5266-4c2d-bb12-70d511485d61","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:21.142Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"94dc2946-5266-4c2d-bb12-70d511485d61","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:21.142Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"94dc2946-5266-4c2d-bb12-70d511485d61","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:21.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"54114266-d329-46cc-9498-c2c281e411a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:21.331Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"54114266-d329-46cc-9498-c2c281e411a3","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:21.331Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"54114266-d329-46cc-9498-c2c281e411a3","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:21.331Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"54114266-d329-46cc-9498-c2c281e411a3"}
{"level":"info","ts":"2026-07-10T10:40:21.331Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"54114266-d329-46cc-9498-c2c281e411a3","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b"}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b"}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:23.119Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"3963fde6-8959-4be9-b851-b5c391afe21b","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef"}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef"}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:40:23.570Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"6b6454c5-810a-4dbd-b2c3-2ed331c551ef","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:23.582Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2b385332-0c86-4bbe-bef1-ae85f6633ab5","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:23.582Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"e519650f-f53d-4783-94a4-9df0f9567fec","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41f7dce8-b189-4a4c-b0d0-6656836b22ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41f7dce8-b189-4a4c-b0d0-6656836b22ac","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"e519650f-f53d-4783-94a4-9df0f9567fec","pod":"stop-klvsc-head-m2plk","ip":"10.133.0.34"}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"e519650f-f53d-4783-94a4-9df0f9567fec","pod":"stop-klvsc-small-group-worker-tzdvk","ip":"10.132.0.25"}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41f7dce8-b189-4a4c-b0d0-6656836b22ac","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"e519650f-f53d-4783-94a4-9df0f9567fec","rayCluster":"stop-klvsc","podIPs":["10.133.0.34","10.132.0.25"]}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41f7dce8-b189-4a4c-b0d0-6656836b22ac"}
{"level":"info","ts":"2026-07-10T10:40:23.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41f7dce8-b189-4a4c-b0d0-6656836b22ac","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:23.595Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2b385332-0c86-4bbe-bef1-ae85f6633ab5","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:23.606Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2b385332-0c86-4bbe-bef1-ae85f6633ab5","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:23.606Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"2b385332-0c86-4bbe-bef1-ae85f6633ab5","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:24.332Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a911de2c-155f-4b8a-bfd4-2ff2681c603c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:24.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a911de2c-155f-4b8a-bfd4-2ff2681c603c","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:24.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a911de2c-155f-4b8a-bfd4-2ff2681c603c","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:24.333Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a911de2c-155f-4b8a-bfd4-2ff2681c603c"}
{"level":"info","ts":"2026-07-10T10:40:24.333Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a911de2c-155f-4b8a-bfd4-2ff2681c603c","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19"}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19"}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","podName":"stop-klvsc-small-group-worker-tzdvk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:25.583Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"2ce1e710-e699-40d6-a308-3f6da35e5e19","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:27.333Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41cd7431-1e9f-4986-8746-d50089b1c3fd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:27.333Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41cd7431-1e9f-4986-8746-d50089b1c3fd","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:27.333Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41cd7431-1e9f-4986-8746-d50089b1c3fd","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:27.333Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41cd7431-1e9f-4986-8746-d50089b1c3fd"}
{"level":"info","ts":"2026-07-10T10:40:27.333Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"41cd7431-1e9f-4986-8746-d50089b1c3fd","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:30.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59d69c19-9fd5-4cb4-807e-c7070487955e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:30.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59d69c19-9fd5-4cb4-807e-c7070487955e","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:30.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59d69c19-9fd5-4cb4-807e-c7070487955e","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:30.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59d69c19-9fd5-4cb4-807e-c7070487955e"}
{"level":"info","ts":"2026-07-10T10:40:30.334Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"59d69c19-9fd5-4cb4-807e-c7070487955e","RayCluster":"stop-klvsc","State":""}
{"level":"info","ts":"2026-07-10T10:40:30.737Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"1af74e1f-80af-4724-8ebb-48e8599a2a72","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:30.737Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"1af74e1f-80af-4724-8ebb-48e8599a2a72","pod":"stop-klvsc-head-m2plk","ip":"10.133.0.34"}
{"level":"info","ts":"2026-07-10T10:40:30.737Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"1af74e1f-80af-4724-8ebb-48e8599a2a72","pod":"stop-klvsc-small-group-worker-tzdvk","ip":"10.132.0.25"}
{"level":"info","ts":"2026-07-10T10:40:30.737Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"1af74e1f-80af-4724-8ebb-48e8599a2a72","rayCluster":"stop-klvsc","podIPs":["10.133.0.34","10.132.0.25"]}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b"}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b"}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:40:32.130Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"8f37c661-c279-4113-807b-fceaa6e37e1b","name":"stop-klvsc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:32.143Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"0691e1ac-4eb5-4071-9171-fddd3c4dd914","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:32.143Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"88e21373-007b-4b71-b5d7-2d6ce80c4996","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"0691e1ac-4eb5-4071-9171-fddd3c4dd914","pod":"stop-klvsc-small-group-worker-tzdvk","ip":"10.132.0.25"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"0691e1ac-4eb5-4071-9171-fddd3c4dd914","pod":"stop-klvsc-head-m2plk","ip":"10.133.0.34"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"0691e1ac-4eb5-4071-9171-fddd3c4dd914","rayCluster":"stop-klvsc","podIPs":["10.132.0.25","10.133.0.34"]}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","head service name":"stop-klvsc-head-svc","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","head service URL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","SubmissionMode":"HTTPMode","RayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:40:32.144Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"35eb5417-c1c9-410b-8101-430022cec583","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:40:32.156Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"51e7bfc9-a7a5-4244-be3b-4a9721977a4c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:32.156Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"51e7bfc9-a7a5-4244-be3b-4a9721977a4c","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:32.156Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"51e7bfc9-a7a5-4244-be3b-4a9721977a4c","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:32.156Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"51e7bfc9-a7a5-4244-be3b-4a9721977a4c"}
{"level":"info","ts":"2026-07-10T10:40:32.162Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"88e21373-007b-4b71-b5d7-2d6ce80c4996","name":"stop-klvsc-head"}
{"level":"info","ts":"2026-07-10T10:40:32.228Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"51e7bfc9-a7a5-4244-be3b-4a9721977a4c","JobId":"stop-hpsnz"}
{"level":"info","ts":"2026-07-10T10:40:32.228Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"51e7bfc9-a7a5-4244-be3b-4a9721977a4c","rayJob":"stop","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/stop.py\",\"submission_id\":\"stop-hpsnz\"}"}
{"level":"info","ts":"2026-07-10T10:40:32.230Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"88e21373-007b-4b71-b5d7-2d6ce80c4996","name":"stop-klvsc-workers"}
{"level":"info","ts":"2026-07-10T10:40:32.230Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"88e21373-007b-4b71-b5d7-2d6ce80c4996","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:33.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7fac5c55-9c8a-479d-ba41-b445c7162af8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:33.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7fac5c55-9c8a-479d-ba41-b445c7162af8","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:33.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7fac5c55-9c8a-479d-ba41-b445c7162af8","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:33.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7fac5c55-9c8a-479d-ba41-b445c7162af8"}
{"level":"info","ts":"2026-07-10T10:40:33.339Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7fac5c55-9c8a-479d-ba41-b445c7162af8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:33.339Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7fac5c55-9c8a-479d-ba41-b445c7162af8","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:40:33.360Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ede4a6ba-39cc-491a-81e7-11628bebd9c1","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:33.360Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ede4a6ba-39cc-491a-81e7-11628bebd9c1","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:33.360Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ede4a6ba-39cc-491a-81e7-11628bebd9c1","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:33.360Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ede4a6ba-39cc-491a-81e7-11628bebd9c1"}
{"level":"info","ts":"2026-07-10T10:40:33.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ede4a6ba-39cc-491a-81e7-11628bebd9c1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:34.144Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e"}
{"level":"info","ts":"2026-07-10T10:40:34.144Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e"}
{"level":"info","ts":"2026-07-10T10:40:34.144Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","1 head service found":"stop-klvsc-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:34.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","Found 1 head Pod":"stop-klvsc-head-m2plk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:40:34.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","head Pod":"stop-klvsc-head-m2plk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-klvsc-head-m2plk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:34.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:40:34.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","worker Pod":"stop-klvsc-small-group-worker-tzdvk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-klvsc-small-group-worker-tzdvk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:40:34.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:34.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:40:34.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:40:34.145Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:40:34.145Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"026c3ddd-63d2-42d3-93e3-75aaeb70ef4e","seconds":300}
{"level":"info","ts":"2026-07-10T10:40:36.360Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81d6f48c-d742-4307-b369-5924368d0e4a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:36.361Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81d6f48c-d742-4307-b369-5924368d0e4a","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:36.361Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81d6f48c-d742-4307-b369-5924368d0e4a","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:36.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81d6f48c-d742-4307-b369-5924368d0e4a"}
{"level":"info","ts":"2026-07-10T10:40:36.364Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81d6f48c-d742-4307-b369-5924368d0e4a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:36.364Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81d6f48c-d742-4307-b369-5924368d0e4a","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:40:36.380Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"5b962731-4013-430c-8031-637d8b36ea18","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:36.380Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"5b962731-4013-430c-8031-637d8b36ea18","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:36.380Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"5b962731-4013-430c-8031-637d8b36ea18","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:36.380Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"5b962731-4013-430c-8031-637d8b36ea18"}
{"level":"info","ts":"2026-07-10T10:40:36.382Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"5b962731-4013-430c-8031-637d8b36ea18","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:39.379Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a3c2422d-d973-45ce-b2ca-f32d533055c8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:39.379Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a3c2422d-d973-45ce-b2ca-f32d533055c8","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:39.379Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a3c2422d-d973-45ce-b2ca-f32d533055c8","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:39.379Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a3c2422d-d973-45ce-b2ca-f32d533055c8"}
{"level":"info","ts":"2026-07-10T10:40:39.382Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"a3c2422d-d973-45ce-b2ca-f32d533055c8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:42.382Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7cc37e12-3dc7-4ad8-b2eb-257bf9574695","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:42.382Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7cc37e12-3dc7-4ad8-b2eb-257bf9574695","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:42.382Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7cc37e12-3dc7-4ad8-b2eb-257bf9574695","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:42.383Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7cc37e12-3dc7-4ad8-b2eb-257bf9574695"}
{"level":"info","ts":"2026-07-10T10:40:42.386Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7cc37e12-3dc7-4ad8-b2eb-257bf9574695","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:45.386Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"337c78d7-021f-4e16-a6e0-3b85f5280cb8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:45.386Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"337c78d7-021f-4e16-a6e0-3b85f5280cb8","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:45.386Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"337c78d7-021f-4e16-a6e0-3b85f5280cb8","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:45.386Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"337c78d7-021f-4e16-a6e0-3b85f5280cb8"}
{"level":"info","ts":"2026-07-10T10:40:45.389Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"337c78d7-021f-4e16-a6e0-3b85f5280cb8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:48.390Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"e2b78b1e-bdb7-4409-bfd7-9b08fae897bf","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:48.390Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"e2b78b1e-bdb7-4409-bfd7-9b08fae897bf","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:48.390Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"e2b78b1e-bdb7-4409-bfd7-9b08fae897bf","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:48.391Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"e2b78b1e-bdb7-4409-bfd7-9b08fae897bf"}
{"level":"info","ts":"2026-07-10T10:40:48.393Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"e2b78b1e-bdb7-4409-bfd7-9b08fae897bf","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:51.394Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81ec2df1-950e-4b55-810f-e7fe67b3c10b","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:51.394Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81ec2df1-950e-4b55-810f-e7fe67b3c10b","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:51.394Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81ec2df1-950e-4b55-810f-e7fe67b3c10b","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:51.394Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81ec2df1-950e-4b55-810f-e7fe67b3c10b"}
{"level":"info","ts":"2026-07-10T10:40:51.397Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"81ec2df1-950e-4b55-810f-e7fe67b3c10b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:54.398Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c8d625da-f5b9-4780-8b82-acb53efb85c9","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:54.398Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c8d625da-f5b9-4780-8b82-acb53efb85c9","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:54.398Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c8d625da-f5b9-4780-8b82-acb53efb85c9","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:54.398Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c8d625da-f5b9-4780-8b82-acb53efb85c9"}
{"level":"info","ts":"2026-07-10T10:40:54.401Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c8d625da-f5b9-4780-8b82-acb53efb85c9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:56.253Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c09a3817-52f9-432b-9815-28d32f41b585","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:40:56.253Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c09a3817-52f9-432b-9815-28d32f41b585","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:40:56.253Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c09a3817-52f9-432b-9815-28d32f41b585","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:40:56.253Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c09a3817-52f9-432b-9815-28d32f41b585","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:40:57.401Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"663e0418-b1c3-4ccf-bd43-df8c8f0ca21f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:57.401Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"663e0418-b1c3-4ccf-bd43-df8c8f0ca21f","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.401Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"663e0418-b1c3-4ccf-bd43-df8c8f0ca21f","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:57.401Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"663e0418-b1c3-4ccf-bd43-df8c8f0ca21f"}
{"level":"info","ts":"2026-07-10T10:40:57.404Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"663e0418-b1c3-4ccf-bd43-df8c8f0ca21f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:40:32Z","endTime":"2026-07-10T10:40:55Z"},"jobId":"stop-hpsnz","rayClusterName":"stop-klvsc","dashboardURL":"stop-klvsc-head-svc.test-ns-7lp9t.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:32Z","stateTransitionTimes":{"ready":"2026-07-10T10:40:32Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.34","serviceIP":"10.133.0.34","podName":"stop-klvsc-head-m2plk","serviceName":"stop-klvsc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:40:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:40:32Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:40:57.404Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"663e0418-b1c3-4ccf-bd43-df8c8f0ca21f","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-10T10:40:57.423Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"e0779f17-fca6-45b4-9b9b-a5a2d1b77f6d","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:40:57.423Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"e0779f17-fca6-45b4-9b9b-a5a2d1b77f6d","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:40:57 +0000 UTC","Now":"2026-07-10T10:40:57.423Z","ShutdownTime":"2026-07-10T10:40:57.000Z"}
{"level":"info","ts":"2026-07-10T10:40:57.650Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"d69f8eb0-6baa-4ce4-8282-8a850f9ea2c9","DeletionTimestamp":"2026-07-10 10:40:57 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:40:57.650Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"d69f8eb0-6baa-4ce4-8282-8a850f9ea2c9","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:40:57.667Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"7c870b85-39ba-4516-a63d-59931bd12506"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"19475981-c366-4571-9ce7-aabfd136db50"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c0271713-bb31-4256-ad4b-3e10c3d3dce5","rayCluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"reconcileID":"53b843c8-f5eb-407d-9d1a-8ba53e5e56be"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"ece836bb-1146-4eca-b88b-d2db2e740792"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","namespace":"test-ns-7lp9t","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","name":"kuberay-gateway-access","namespace":"test-ns-7lp9t","remainingClusters":1}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c0271713-bb31-4256-ad4b-3e10c3d3dce5","pod":"stop-klvsc-small-group-worker-tzdvk","ip":"10.132.0.25"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c0271713-bb31-4256-ad4b-3e10c3d3dce5","pod":"stop-klvsc-head-m2plk","ip":"10.133.0.34"}
{"level":"info","ts":"2026-07-10T10:40:57.692Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"c0271713-bb31-4256-ad4b-3e10c3d3dce5","rayCluster":"stop-klvsc","podIPs":["10.132.0.25","10.133.0.34"]}
{"level":"info","ts":"2026-07-10T10:40:57.700Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","name":"test-ns-7lp9t-stop-klvsc","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:40:57.707Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","configMap":"kube-rbac-proxy-config-stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.714Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","serviceAccount":"stop-klvsc-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:40:57.733Z","logger":"raycluster-resource","msg":"default","name":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.733Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-klvsc","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"da8f59ed-b8be-4b76-8de1-e20e8c523fab","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169","cluster":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169","namespace":"test-ns-7lp9t","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"14cbc5dc-6860-44cd-b9d4-6563ce55387e"}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169","name":"kuberay-gateway-access","namespace":"test-ns-7lp9t","remainingClusters":1}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"98745fc2-a471-4719-935a-0a437b250ac2"}
{"level":"info","ts":"2026-07-10T10:40:57.745Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bb4b28e6-df9e-47c3-b316-80719bb7b0d9","namespace":"test-ns-7lp9t","clusterName":"stop-klvsc"}
{"level":"error","ts":"2026-07-10T10:40:57.750Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169","error":"rayclusters.ray.io \"stop-klvsc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:57.750Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169"}
{"level":"error","ts":"2026-07-10T10:40:57.750Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9c4f36b2-f1ab-49f0-b6f7-43a12898f169","error":"rayclusters.ray.io \"stop-klvsc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:57.750Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"97153196-08d4-4931-8b2b-18fe1cd12638","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:57.750Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"97153196-08d4-4931-8b2b-18fe1cd12638"}
{"level":"info","ts":"2026-07-10T10:40:57.750Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"97153196-08d4-4931-8b2b-18fe1cd12638","namespace":"test-ns-7lp9t","authClusters":1}
{"level":"info","ts":"2026-07-10T10:40:57.750Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"97153196-08d4-4931-8b2b-18fe1cd12638","namespace":"test-ns-7lp9t","authClusters":1}
{"level":"info","ts":"2026-07-10T10:40:57.755Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"217a6f9d-a757-48a0-a8c2-67f84da4d661","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:57.755Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"217a6f9d-a757-48a0-a8c2-67f84da4d661"}
{"level":"info","ts":"2026-07-10T10:40:57.755Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"217a6f9d-a757-48a0-a8c2-67f84da4d661","namespace":"test-ns-7lp9t","authClusters":1}
{"level":"info","ts":"2026-07-10T10:40:57.755Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"217a6f9d-a757-48a0-a8c2-67f84da4d661","namespace":"test-ns-7lp9t","authClusters":1}
{"level":"info","ts":"2026-07-10T10:40:57.757Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"3f610faf-84ca-4e6c-b1bf-458f0d8ab43a"}
{"level":"info","ts":"2026-07-10T10:40:57.758Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9bbc2d31-5b3f-42bc-bd09-7ef5c00a40be","namespacedName":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:40:57.758Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9bbc2d31-5b3f-42bc-bd09-7ef5c00a40be"}
{"level":"info","ts":"2026-07-10T10:40:57.758Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9bbc2d31-5b3f-42bc-bd09-7ef5c00a40be","namespace":"test-ns-7lp9t","authClusters":1}
{"level":"info","ts":"2026-07-10T10:40:57.758Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"9bbc2d31-5b3f-42bc-bd09-7ef5c00a40be","namespace":"test-ns-7lp9t","authClusters":1}
{"level":"error","ts":"2026-07-10T10:40:57.770Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bb4b28e6-df9e-47c3-b316-80719bb7b0d9","certificate":"ray-head-cert-stop-klvsc","error":"certificates.cert-manager.io \"ray-head-cert-stop-klvsc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:57.770Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"f258e43f-150d-4077-9826-7ef0a22b4a9a"}
{"level":"info","ts":"2026-07-10T10:40:57.825Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d215f75d-cbfa-4aee-a9fc-6a069caa12dc","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:40:57.825Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d215f75d-cbfa-4aee-a9fc-6a069caa12dc","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:40:57.836Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bb4b28e6-df9e-47c3-b316-80719bb7b0d9","errorCount":1}
{"level":"info","ts":"2026-07-10T10:40:57.836Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bb4b28e6-df9e-47c3-b316-80719bb7b0d9"}
{"level":"error","ts":"2026-07-10T10:40:57.836Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"bb4b28e6-df9e-47c3-b316-80719bb7b0d9","error":"certificates.cert-manager.io \"ray-head-cert-stop-klvsc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:57.839Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d215f75d-cbfa-4aee-a9fc-6a069caa12dc"}
{"level":"info","ts":"2026-07-10T10:40:57.839Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d215f75d-cbfa-4aee-a9fc-6a069caa12dc","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:40:57.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d215f75d-cbfa-4aee-a9fc-6a069caa12dc","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:40:57.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d215f75d-cbfa-4aee-a9fc-6a069caa12dc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:40:57.841Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"d6bb70b7-878c-419d-aa97-a89bf50e60c9","namespace":"test-ns-7lp9t","clusterName":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.841Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-klvsc","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"stop-klvsc","reconcileID":"d6bb70b7-878c-419d-aa97-a89bf50e60c9","namespace":"test-ns-7lp9t","clusterName":"stop-klvsc"}
{"level":"info","ts":"2026-07-10T10:40:57.852Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3804e237-d60e-42c9-94c5-3139756c6110","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:40:57.852Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3804e237-d60e-42c9-94c5-3139756c6110","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.852Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3804e237-d60e-42c9-94c5-3139756c6110","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:57.852Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3804e237-d60e-42c9-94c5-3139756c6110","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:57.857Z","logger":"raycluster-resource","msg":"default","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.857Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:40:57.875Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.875Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.875Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357"}
{"level":"info","ts":"2026-07-10T10:40:57.875Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357"}
{"level":"info","ts":"2026-07-10T10:40:57.875Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ced0727c-908d-4e60-a017-e53d4d009aa4","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","mode":"IntegratedOAuth","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3804e237-d60e-42c9-94c5-3139756c6110","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3804e237-d60e-42c9-94c5-3139756c6110"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3804e237-d60e-42c9-94c5-3139756c6110","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"02fe1ab5-5317-420a-87b4-fbd85eab8844","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"02fe1ab5-5317-420a-87b4-fbd85eab8844","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"02fe1ab5-5317-420a-87b4-fbd85eab8844","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"02fe1ab5-5317-420a-87b4-fbd85eab8844"}
{"level":"info","ts":"2026-07-10T10:40:57.876Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"02fe1ab5-5317-420a-87b4-fbd85eab8844","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"raycluster-resource","msg":"default","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","name":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k"}
{"level":"error","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:40:57.925Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","name":"long-running-zpp4k","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:57.932Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ced0727c-908d-4e60-a017-e53d4d009aa4","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.932Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8255fe5e-539d-4b2e-a22a-5967bf22b2b2","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"52cf4fa7-107d-4ff2-ac45-49a117e02a16","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:40:57.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"52cf4fa7-107d-4ff2-ac45-49a117e02a16","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"52cf4fa7-107d-4ff2-ac45-49a117e02a16","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:57.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"52cf4fa7-107d-4ff2-ac45-49a117e02a16"}
{"level":"info","ts":"2026-07-10T10:40:57.936Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"52cf4fa7-107d-4ff2-ac45-49a117e02a16","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:40:57.939Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:40:57.939Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:57.943Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","name":"long-running-zpp4k-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:40:57.943Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","name":"long-running-zpp4k","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-zpp4k\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"long-running-zpp4k"}}
{"level":"info","ts":"2026-07-10T10:40:57.943Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357"}
{"level":"error","ts":"2026-07-10T10:40:57.943Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6ab3bded-88d8-4122-b87a-40d832fb2357","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:57.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0"}
{"level":"info","ts":"2026-07-10T10:40:57.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0"}
{"level":"info","ts":"2026-07-10T10:40:57.943Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:40:57.944Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","name":"long-running-zpp4k","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:40:57.950Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8255fe5e-539d-4b2e-a22a-5967bf22b2b2","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:40:57.951Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","name":"kuberay-gateway-access","namespace":"test-ns-pqpm4","operation":"created"}
{"level":"info","ts":"2026-07-10T10:40:58.025Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.025Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","name":"test-ns-pqpm4-long-running-zpp4k","operation":"created"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0"}
{"level":"error","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"129145b2-531c-479f-a504-cc43623962f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"615c3fb9-f54e-4279-a70d-fa45b7f336e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"615c3fb9-f54e-4279-a70d-fa45b7f336e1","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"615c3fb9-f54e-4279-a70d-fa45b7f336e1","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"615c3fb9-f54e-4279-a70d-fa45b7f336e1"}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"615c3fb9-f54e-4279-a70d-fa45b7f336e1","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77"}
{"level":"error","ts":"2026-07-10T10:40:58.026Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"16984122-2c9f-4d19-8d9f-d60aeb202b77","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.034Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","name":"kube-rbac-proxy-config-long-running-zpp4k","operation":"created"}
{"level":"info","ts":"2026-07-10T10:40:58.034Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8255fe5e-539d-4b2e-a22a-5967bf22b2b2","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:40:58.034Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8255fe5e-539d-4b2e-a22a-5967bf22b2b2","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.034Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8ee6be3b-ee17-4c07-b0da-f478f2ed4e30","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1"}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1"}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1"}
{"level":"error","ts":"2026-07-10T10:40:58.036Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f19a6be0-23cd-418b-8598-85466f2949c1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.040Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:40:58.042Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-zpp4k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.042Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446"}
{"level":"error","ts":"2026-07-10T10:40:58.042Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8b3a6fed-5635-4a7d-a972-14c842429446","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-zpp4k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.042Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9da707ed-07de-4124-af51-d73d068b9153","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:58.042Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9da707ed-07de-4124-af51-d73d068b9153","mode":"IntegratedOAuth","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.042Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9da707ed-07de-4124-af51-d73d068b9153","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.048Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8ee6be3b-ee17-4c07-b0da-f478f2ed4e30","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235","rayCluster":"long-running-zpp4k","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a5ef6962-757d-406f-bcae-7e6104f9d235"}
{"level":"error","ts":"2026-07-10T10:40:58.125Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"62e73f81-7a40-40a9-b923-e35223d06289","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a2336b0a-6af4-418e-a855-661b0257f66a","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"a2336b0a-6af4-418e-a855-661b0257f66a"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9da707ed-07de-4124-af51-d73d068b9153","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"dfda5f3e-5fea-45cd-a4d9-d92da3013a50","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"dfda5f3e-5fea-45cd-a4d9-d92da3013a50","mode":"IntegratedOAuth","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"dfda5f3e-5fea-45cd-a4d9-d92da3013a50","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.125Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7c2fc039-8053-4fed-9936-7e5ab6079434","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.126Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7c2fc039-8053-4fed-9936-7e5ab6079434"}
{"level":"info","ts":"2026-07-10T10:40:58.126Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ec155b5-f279-4fd1-a275-8c780b99a969","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:40:58.126Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ec155b5-f279-4fd1-a275-8c780b99a969","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.126Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ec155b5-f279-4fd1-a275-8c780b99a969","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:40:58.126Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ec155b5-f279-4fd1-a275-8c780b99a969"}
{"level":"info","ts":"2026-07-10T10:40:58.126Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ec155b5-f279-4fd1-a275-8c780b99a969","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:40:58.130Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8ee6be3b-ee17-4c07-b0da-f478f2ed4e30","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:40:58.130Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"8ee6be3b-ee17-4c07-b0da-f478f2ed4e30","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.130Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"289095a9-3c8c-45ef-a9dc-a24e10c3a9da","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.136Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"dfda5f3e-5fea-45cd-a4d9-d92da3013a50","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.142Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"289095a9-3c8c-45ef-a9dc-a24e10c3a9da","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:40:58.152Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"289095a9-3c8c-45ef-a9dc-a24e10c3a9da","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:40:58.152Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"289095a9-3c8c-45ef-a9dc-a24e10c3a9da","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba"}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba"}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba"}
{"level":"error","ts":"2026-07-10T10:40:58.225Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"de531635-3005-4903-901e-2a7eda19daba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.385Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7"}
{"level":"info","ts":"2026-07-10T10:40:58.385Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7"}
{"level":"info","ts":"2026-07-10T10:40:58.385Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:58.385Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:58.385Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.386Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:58.386Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.386Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.386Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.386Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7"}
{"level":"error","ts":"2026-07-10T10:40:58.386Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8c28210-9948-436b-9cbd-6d1a7b2f9fc7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659"}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659"}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","error":"head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659"}
{"level":"error","ts":"2026-07-10T10:40:58.706Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c929e3d9-1310-43da-9fbd-2befcd6de659","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:59.347Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43"}
{"level":"info","ts":"2026-07-10T10:40:59.347Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43"}
{"level":"info","ts":"2026-07-10T10:40:59.347Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","error":"worker certificate secret ray-worker-secret-long-running-zpp4k not found"}
{"level":"error","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-zpp4k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-zpp4k not found"}
{"level":"info","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-zpp4k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43"}
{"level":"error","ts":"2026-07-10T10:40:59.348Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"91d3fa35-086d-419f-8ffd-041f2763df43","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-zpp4k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:00.423Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-7lp9t"},"reconcileID":"c2a2bd45-69ae-4b9a-969f-aae63926ceee"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","mode":"IntegratedOAuth","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","cluster":"long-running-zpp4k","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","cluster":"long-running-zpp4k","authType":"IntegratedOAuth","serviceAccount":"long-running-zpp4k-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-zpp4k","ray.io/group":"headgroup","ray.io/identifier":"long-running-zpp4k-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:00.629Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","enabled":true}
{"level":"info","ts":"2026-07-10T10:41:00.647Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","name":"long-running-zpp4k-head-kcpg5"}
{"level":"info","ts":"2026-07-10T10:41:00.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af"}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","nodeType":"worker","rayStartParams":{"address":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","rayStartCmd":"ray start  --address=long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:00.648Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","enabled":true}
{"level":"info","ts":"2026-07-10T10:41:00.661Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","name":"long-running-zpp4k-small-group-worker-l4bv9"}
{"level":"info","ts":"2026-07-10T10:41:00.661Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.661Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-zpp4k-head-svc"},"newHead":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"}}
{"level":"info","ts":"2026-07-10T10:41:00.662Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"363377f1-ac64-4a93-a82d-955032ba74af","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f1b514d-69c3-4b8b-998f-55e33ecda3c7","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"d3c83121-f829-42dc-9f70-758078038a4f","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5319ff1c-ce93-4d82-8954-c23edce3edc1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5319ff1c-ce93-4d82-8954-c23edce3edc1","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5319ff1c-ce93-4d82-8954-c23edce3edc1","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-zpp4k-head-svc"},"newHead":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"}}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"d3c83121-f829-42dc-9f70-758078038a4f","rayCluster":"long-running-zpp4k","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:40:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5319ff1c-ce93-4d82-8954-c23edce3edc1"}
{"level":"info","ts":"2026-07-10T10:41:00.674Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5319ff1c-ce93-4d82-8954-c23edce3edc1","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:00.685Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","name":"long-running-zpp4k","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-zpp4k\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"long-running-zpp4k"}}
{"level":"info","ts":"2026-07-10T10:41:00.685Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b"}
{"level":"error","ts":"2026-07-10T10:41:00.685Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"47080dad-f58b-4b0d-b9c5-f813eedc0f7b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-zpp4k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:00.685Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd"}
{"level":"info","ts":"2026-07-10T10:41:00.686Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd"}
{"level":"info","ts":"2026-07-10T10:41:00.686Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:00.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:00.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:00.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:00.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.689Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f1b514d-69c3-4b8b-998f-55e33ecda3c7","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:41:00.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","podName":"long-running-zpp4k-small-group-worker-l4bv9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","podName":"long-running-zpp4k-small-group-worker-l4bv9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.725Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:41:00.725Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"5d54d6c7-76c1-4529-ab0e-750f38c927cd","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"b47d04c6-9058-49ba-a062-ae025450f9a7","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"b47d04c6-9058-49ba-a062-ae025450f9a7","rayCluster":"long-running-zpp4k","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"dd6b417f-dbea-458c-8c37-885d5be53d6e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"dd6b417f-dbea-458c-8c37-885d5be53d6e","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"dd6b417f-dbea-458c-8c37-885d5be53d6e","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:41:00.738Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:00.739Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f1b514d-69c3-4b8b-998f-55e33ecda3c7","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:41:00.739Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"dd6b417f-dbea-458c-8c37-885d5be53d6e"}
{"level":"info","ts":"2026-07-10T10:41:00.739Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"dd6b417f-dbea-458c-8c37-885d5be53d6e","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:00.739Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f1b514d-69c3-4b8b-998f-55e33ecda3c7","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.739Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"885fb6eb-bf60-4745-9890-252aaeeccae1","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.751Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"885fb6eb-bf60-4745-9890-252aaeeccae1","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:41:00.761Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"885fb6eb-bf60-4745-9890-252aaeeccae1","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:41:00.761Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"885fb6eb-bf60-4745-9890-252aaeeccae1","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.832Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","name":"long-running-zpp4k","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-zpp4k\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"long-running-zpp4k"}}
{"level":"info","ts":"2026-07-10T10:41:00.832Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb"}
{"level":"error","ts":"2026-07-10T10:41:00.832Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"dc4e17eb-eaee-4851-a42f-9297fd8a3dcb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-zpp4k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:00.837Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d"}
{"level":"info","ts":"2026-07-10T10:41:00.837Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d"}
{"level":"info","ts":"2026-07-10T10:41:00.837Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:00.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:00.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:00.838Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f6cf34cc-62ad-43a4-a12e-56d5878e5c8d","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:00.853Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0516a23b-92dd-41f1-a9c2-d279e0210d98","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:00.853Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0516a23b-92dd-41f1-a9c2-d279e0210d98","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:00.853Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0516a23b-92dd-41f1-a9c2-d279e0210d98","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:00.853Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0516a23b-92dd-41f1-a9c2-d279e0210d98"}
{"level":"info","ts":"2026-07-10T10:41:00.853Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0516a23b-92dd-41f1-a9c2-d279e0210d98","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8"}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8"}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.098Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.099Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:01.099Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bc3914cf-ef0f-48b1-bcd7-d001257088b8","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:01.121Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5"}
{"level":"info","ts":"2026-07-10T10:41:01.121Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5"}
{"level":"info","ts":"2026-07-10T10:41:01.121Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:01.122Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2c0e398b-e307-4a06-b462-6422bb368ea5","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2"}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2"}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.271Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.272Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:01.272Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e5f02f9f-f6b0-47da-aae5-41ace18c99f2","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"newHead":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"}}
{"level":"info","ts":"2026-07-10T10:41:01.770Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f8a75528-a0e0-4bae-b8d2-9ffbe0243c99","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:01.782Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7dfc75bb-abdf-4b35-9e6d-d2dead716f6a","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:01.782Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"81bb276c-aee9-4744-94d2-61413b26364f","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:01.783Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7dfc75bb-abdf-4b35-9e6d-d2dead716f6a","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:41:01.783Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7dfc75bb-abdf-4b35-9e6d-d2dead716f6a","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:41:01.783Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"bd3d4529-3b3d-4149-b16d-1f66ef8c68f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:01.783Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"bd3d4529-3b3d-4149-b16d-1f66ef8c68f7","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:01.783Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"bd3d4529-3b3d-4149-b16d-1f66ef8c68f7","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:01.783Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"bd3d4529-3b3d-4149-b16d-1f66ef8c68f7"}
{"level":"info","ts":"2026-07-10T10:41:01.783Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"bd3d4529-3b3d-4149-b16d-1f66ef8c68f7","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:01.795Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7dfc75bb-abdf-4b35-9e6d-d2dead716f6a","certificate":"ray-head-cert-long-running-zpp4k","dnsNames":["long-running-zpp4k-head-svc","localhost","long-running-zpp4k-head-svc.test-ns-pqpm4.svc","long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local"],"ipAddresses":["10.133.0.35","10.132.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:41:01.796Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"81bb276c-aee9-4744-94d2-61413b26364f","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:41:01.806Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"81bb276c-aee9-4744-94d2-61413b26364f","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:41:01.806Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"81bb276c-aee9-4744-94d2-61413b26364f","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:01.807Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7dfc75bb-abdf-4b35-9e6d-d2dead716f6a","certificate":"ray-worker-cert-long-running-zpp4k","dnsNames":["long-running-zpp4k-worker-svc","localhost","long-running-zpp4k-small-group","long-running-zpp4k-worker-svc.test-ns-pqpm4.svc","long-running-zpp4k-worker-svc.test-ns-pqpm4.svc.cluster.local","*.long-running-zpp4k-worker-svc.test-ns-pqpm4.svc","*.long-running-zpp4k-worker-svc.test-ns-pqpm4.svc.cluster.local","*-worker-*.test-ns-pqpm4.svc","*-worker-*.test-ns-pqpm4.svc.cluster.local"],"ipAddresses":["10.133.0.35","10.132.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:41:01.807Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"7dfc75bb-abdf-4b35-9e6d-d2dead716f6a","rayCluster":"long-running-zpp4k","podIPs":["10.133.0.35","10.132.0.26"]}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9"}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9"}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:02.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.794Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:02.794Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"57595d7c-3776-4b7d-9bb2-d41d541830a9","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1"}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1"}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:02.802Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"a1820a4a-04e5-4a1e-b0d0-080555a88dc1","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:02.820Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d"}
{"level":"info","ts":"2026-07-10T10:41:02.820Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d"}
{"level":"info","ts":"2026-07-10T10:41:02.820Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:02.821Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"d526463c-3cc1-4948-87aa-b4d1dfb59c7d","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3"}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3"}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-rkvnj-small-group-worker-qv8sz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:02.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:02.826Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:02.826Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"059290e0-92ef-4dde-a686-ac681a8256e3","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:03.368Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5"}
{"level":"info","ts":"2026-07-10T10:41:03.368Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5"}
{"level":"info","ts":"2026-07-10T10:41:03.368Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:03.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:03.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","worker Pod":"fail-rkvnj-small-group-worker-qv8sz","shouldDelete":true,"reason":"The worker Pod fail-rkvnj-small-group-worker-qv8sz status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T10:41:03.372Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.372Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-07-10T10:41:03.373Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.373Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","name":"fail-rkvnj","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:39:55Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5"}
{"level":"error","ts":"2026-07-10T10:41:03.385Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"556cb92a-4d5a-426f-ba5d-406280538dd5","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f6c4abd7-1239-4d8f-ad78-e37c20e0065e","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"032c8686-52c1-42f7-ac2a-49062c963d0c","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"103773e2-e9b5-4308-b7c1-cf7c25a57065","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"103773e2-e9b5-4308-b7c1-cf7c25a57065","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:39:59 +0000 UTC","Now":"2026-07-10T10:41:03.385Z","ShutdownTime":"2026-07-10T10:39:59.000Z"}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f6c4abd7-1239-4d8f-ad78-e37c20e0065e","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f6c4abd7-1239-4d8f-ad78-e37c20e0065e","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:41:03.385Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"f6c4abd7-1239-4d8f-ad78-e37c20e0065e","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4"}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4"}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-rkvnj-head-p58jf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4","podName":"fail-rkvnj-small-group-worker-qv8sz","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"032c8686-52c1-42f7-ac2a-49062c963d0c","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:03.390Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5fc81b0b-e5e6-4c24-8b0f-d4cd244909a4","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:03.768Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368"}
{"level":"info","ts":"2026-07-10T10:41:03.768Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368"}
{"level":"info","ts":"2026-07-10T10:41:03.768Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","Found 1 head Pod":"fail-rkvnj-head-p58jf","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-10 10:38:33 +0000 UTC,FinishedAt:2026-07-10 10:41:03 +0000 UTC,ContainerID:cri-o://48062feadb68f65168e65d086d6015581ec8d30af158e553c570245adee65482,}"}
{"level":"info","ts":"2026-07-10T10:41:03.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","head Pod":"fail-rkvnj-head-p58jf","shouldDelete":true,"reason":"The head Pod fail-rkvnj-head-p58jf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T10:41:03.775Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod fail-rkvnj-head-p58jf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.775Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","error":"The head Pod fail-rkvnj-head-p58jf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-10T10:41:03.775Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:41:03.775Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","name":"fail-rkvnj","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:38:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:03.788Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368"}
{"level":"error","ts":"2026-07-10T10:41:03.788Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"7c4cf457-8812-49df-92be-c3c3fc3cb368","error":"The head Pod fail-rkvnj-head-p58jf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"8185a226-13ed-4343-a399-f66a57d0f77a","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"ad90c61f-d170-4b2c-b98b-36f46c4fab0c","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6657cc15-52c0-498c-93bf-39b940005ec0","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"6657cc15-52c0-498c-93bf-39b940005ec0","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:39:59 +0000 UTC","Now":"2026-07-10T10:41:03.789Z","ShutdownTime":"2026-07-10T10:39:59.000Z"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"8185a226-13ed-4343-a399-f66a57d0f77a","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"8185a226-13ed-4343-a399-f66a57d0f77a","pod":"fail-rkvnj-small-group-worker-qv8sz","ip":"10.133.0.33"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"8185a226-13ed-4343-a399-f66a57d0f77a","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24","10.133.0.33"]}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"bf3e1305-a0fe-4762-8b71-7ac36afbeeae","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde","podName":"fail-rkvnj-head-p58jf","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:03.789Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"3200f20b-0510-403f-b02b-cc0f1a4b9bde","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a"}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a"}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a"}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.791Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","enabled":true}
{"level":"info","ts":"2026-07-10T10:41:03.825Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"ad90c61f-d170-4b2c-b98b-36f46c4fab0c","cluster":"fail-rkvnj"}
{"level":"error","ts":"2026-07-10T10:41:03.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:03.825Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:41:03.825Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","name":"fail-rkvnj","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a"}
{"level":"error","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"8935a13b-83a5-4610-bf41-ae7836d6bd0a","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"187e132b-529d-4a57-9e79-6dc556a28100","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7a823527-1862-4393-8946-78270ecb9064","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"187e132b-529d-4a57-9e79-6dc556a28100","pod":"fail-rkvnj-head-p58jf","ip":"10.132.0.24"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","enabled":true}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0deb985f-2104-40c1-b253-b8cfc1db4431","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:41:03.838Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"0deb985f-2104-40c1-b253-b8cfc1db4431","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:39:59 +0000 UTC","Now":"2026-07-10T10:41:03.838Z","ShutdownTime":"2026-07-10T10:39:59.000Z"}
{"level":"error","ts":"2026-07-10T10:41:03.841Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.841Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:03.842Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:41:03.842Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","name":"fail-rkvnj","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:03.843Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"7a823527-1862-4393-8946-78270ecb9064","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:03.848Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"187e132b-529d-4a57-9e79-6dc556a28100","certificate":"ray-head-cert-fail-rkvnj","dnsNames":["fail-rkvnj-head-svc","localhost","fail-rkvnj-head-svc.test-ns-7lp9t.svc","fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.132.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:41:03.848Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","name":"fail-rkvnj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-rkvnj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"fail-rkvnj"}}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c"}
{"level":"error","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"97193925-9bb0-4cda-abf4-c91e0e61923c","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806"}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.849Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","enabled":true}
{"level":"error","ts":"2026-07-10T10:41:03.852Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.853Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:03.853Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806"}
{"level":"error","ts":"2026-07-10T10:41:03.853Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"61383f02-1bd6-4ebc-8c06-bdd7c2f0c806","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606"}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.859Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","enabled":true}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e52010b9-2709-4e56-bd8c-c547fb98d943","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e52010b9-2709-4e56-bd8c-c547fb98d943","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e52010b9-2709-4e56-bd8c-c547fb98d943","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"error","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"187e132b-529d-4a57-9e79-6dc556a28100","certificate":"ray-worker-cert-fail-rkvnj","dnsNames":["fail-rkvnj-worker-svc","localhost","fail-rkvnj-small-group","fail-rkvnj-worker-svc.test-ns-7lp9t.svc","fail-rkvnj-worker-svc.test-ns-7lp9t.svc.cluster.local","*.fail-rkvnj-worker-svc.test-ns-7lp9t.svc","*.fail-rkvnj-worker-svc.test-ns-7lp9t.svc.cluster.local","*-worker-*.test-ns-7lp9t.svc","*-worker-*.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["10.132.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"187e132b-529d-4a57-9e79-6dc556a28100","rayCluster":"fail-rkvnj","podIPs":["10.132.0.24"]}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e52010b9-2709-4e56-bd8c-c547fb98d943"}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e52010b9-2709-4e56-bd8c-c547fb98d943","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606"}
{"level":"error","ts":"2026-07-10T10:41:03.925Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f976cdcb-f293-4781-b56f-7f5141899606","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355"}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:03.966Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","enabled":true}
{"level":"error","ts":"2026-07-10T10:41:03.971Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:03.971Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:03.971Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355"}
{"level":"error","ts":"2026-07-10T10:41:03.971Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"5dd35b1f-3d8b-47f4-b043-0e03c606c355","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da"}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:04.052Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","enabled":true}
{"level":"error","ts":"2026-07-10T10:41:04.055Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.055Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:04.056Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da"}
{"level":"error","ts":"2026-07-10T10:41:04.056Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2099232b-e990-4663-9ab8-88380221e1da","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1"}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:04.217Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","enabled":true}
{"level":"error","ts":"2026-07-10T10:41:04.221Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.221Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:04.221Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1"}
{"level":"error","ts":"2026-07-10T10:41:04.221Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"51c2a00f-fdb3-4a6a-b32e-004164cfd2a1","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76"}
{"level":"info","ts":"2026-07-10T10:41:04.289Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","nodeType":"worker","rayStartParams":{"address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:41:04.290Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","rayStartCmd":"ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:04.290Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:41:04.290Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","enabled":true}
{"level":"error","ts":"2026-07-10T10:41:04.293Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.293Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:04.293Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.293Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.293Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.293Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"newHead":{"serviceName":"fail-rkvnj-head-svc"}}
{"level":"info","ts":"2026-07-10T10:41:04.293Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","name":"fail-rkvnj","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76"}
{"level":"error","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"f44d3452-1e0d-4c5e-a16f-bd54e7535b76","error":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","cluster":"fail-rkvnj","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","cluster":"fail-rkvnj","authType":"IntegratedOAuth","serviceAccount":"fail-rkvnj-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-rkvnj","ray.io/group":"headgroup","ray.io/identifier":"fail-rkvnj-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","enabled":true}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"60746ebb-0e0a-4eb8-b098-1a4651756ab5","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"6d72cbbd-a3ea-437f-a34d-73bd6ea097d8","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"db6be868-8ade-4595-b356-1005424ef441","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:41:04.306Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"db6be868-8ade-4595-b356-1005424ef441","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:39:59 +0000 UTC","Now":"2026-07-10T10:41:04.306Z","ShutdownTime":"2026-07-10T10:39:59.000Z"}
{"level":"error","ts":"2026-07-10T10:41:04.310Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.310Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:04.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.310Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.310Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"newHead":{"serviceName":"fail-rkvnj-head-svc"}}
{"level":"info","ts":"2026-07-10T10:41:04.310Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","name":"fail-rkvnj","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-rkvnj-head-p58jf","serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:04.312Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"6d72cbbd-a3ea-437f-a34d-73bd6ea097d8","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:04.320Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"60746ebb-0e0a-4eb8-b098-1a4651756ab5","certificate":"ray-head-cert-fail-rkvnj","dnsNames":["fail-rkvnj-head-svc","localhost","fail-rkvnj-head-svc.test-ns-7lp9t.svc","fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:41:04.320Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","name":"fail-rkvnj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-rkvnj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-7lp9t","name":"fail-rkvnj"}}
{"level":"info","ts":"2026-07-10T10:41:04.320Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1"}
{"level":"error","ts":"2026-07-10T10:41:04.320Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"2e8852ae-9068-4f18-ade2-00efef3a9df1","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.333Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"60746ebb-0e0a-4eb8-b098-1a4651756ab5","certificate":"ray-worker-cert-fail-rkvnj","dnsNames":["fail-rkvnj-worker-svc","localhost","fail-rkvnj-small-group","fail-rkvnj-worker-svc.test-ns-7lp9t.svc","fail-rkvnj-worker-svc.test-ns-7lp9t.svc.cluster.local","*.fail-rkvnj-worker-svc.test-ns-7lp9t.svc","*.fail-rkvnj-worker-svc.test-ns-7lp9t.svc.cluster.local","*-worker-*.test-ns-7lp9t.svc","*-worker-*.test-ns-7lp9t.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:41:04.333Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"60746ebb-0e0a-4eb8-b098-1a4651756ab5"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","cluster":"fail-rkvnj","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","cluster":"fail-rkvnj","authType":"IntegratedOAuth","serviceAccount":"fail-rkvnj-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-rkvnj","ray.io/group":"headgroup","ray.io/identifier":"fail-rkvnj-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:04.542Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","enabled":true}
{"level":"error","ts":"2026-07-10T10:41:04.546Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.546Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:04.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:04.547Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:41:04.547Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","name":"fail-rkvnj","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-rkvnj-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-rkvnj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:38:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:38:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:41:03Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:41:04.558Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7"}
{"level":"error","ts":"2026-07-10T10:41:04.558Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"ed181604-b227-4005-a94e-ca904ad1c6b7","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:04.559Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"32eb1d58-36c6-4c4c-8436-cfff61041e47","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:04.559Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"bb898a65-5f4b-49ae-98db-89151a87aaee","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:04.559Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"582c9893-daa1-413f-bd1f-d27ba460fcd5","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-10T10:41:04.559Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"582c9893-daa1-413f-bd1f-d27ba460fcd5","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:39:59 +0000 UTC","Now":"2026-07-10T10:41:04.559Z","ShutdownTime":"2026-07-10T10:39:59.000Z"}
{"level":"info","ts":"2026-07-10T10:41:04.559Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"bb898a65-5f4b-49ae-98db-89151a87aaee"}
{"level":"info","ts":"2026-07-10T10:41:04.564Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"32eb1d58-36c6-4c4c-8436-cfff61041e47","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:06.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31"}
{"level":"info","ts":"2026-07-10T10:41:06.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31"}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:06.301Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"22fe27f5-5ed8-4d23-a144-dfbe5a528d31","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:06.926Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"10a06b2a-264f-43b1-b392-937bf7949a83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:06.926Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"10a06b2a-264f-43b1-b392-937bf7949a83","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:06.926Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"10a06b2a-264f-43b1-b392-937bf7949a83","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:06.926Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"10a06b2a-264f-43b1-b392-937bf7949a83"}
{"level":"info","ts":"2026-07-10T10:41:06.926Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"10a06b2a-264f-43b1-b392-937bf7949a83","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:07.119Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47"}
{"level":"info","ts":"2026-07-10T10:41:07.119Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47"}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","1 head service found":"fail-rkvnj-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47"}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47"}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","cluster":"fail-rkvnj","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","cluster":"fail-rkvnj","authType":"IntegratedOAuth","serviceAccount":"fail-rkvnj-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-rkvnj","ray.io/group":"headgroup","ray.io/identifier":"fail-rkvnj-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-rkvnj-head-svc.test-ns-7lp9t.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:41:07.120Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","enabled":true}
{"level":"error","ts":"2026-07-10T10:41:07.124Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:07.125Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:07.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:07.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:07.126Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:07.126Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47"}
{"level":"error","ts":"2026-07-10T10:41:07.126Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"6c963dfd-f59a-4ccd-b637-f22f7921ca47","error":"FailedCreateHeadPod\npods \"fail-rkvnj-head-\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:41:07.306Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"a81d420f-7ad6-4496-9e44-d75348460fa6","seconds":300}
{"level":"info","ts":"2026-07-10T10:41:08.665Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9a61939c-8d9d-4c58-9453-24777c1a5bb0","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:41:08.665Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9a61939c-8d9d-4c58-9453-24777c1a5bb0","mode":"IntegratedOAuth","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:08.665Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9a61939c-8d9d-4c58-9453-24777c1a5bb0","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:08.665Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606"}
{"level":"info","ts":"2026-07-10T10:41:08.665Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606"}
{"level":"error","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"fail-rkvnj-head-svc\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606","error":"services \"fail-rkvnj-head-svc\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606","serviceSelectors":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/identifier":"fail-rkvnj-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606","filter labels":["test-ns-7lp9t",{"ray.io/cluster":"fail-rkvnj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606","error":"unable to find head service. cluster name fail-rkvnj, filter labels [test-ns-7lp9t map[ray.io/cluster:fail-rkvnj ray.io/identifier:fail-rkvnj-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606"}
{"level":"error","ts":"2026-07-10T10:41:08.668Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"43ea70c7-e5d7-495e-abe9-462d27b5e606","error":"services \"fail-rkvnj-head-svc\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:08.675Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9a61939c-8d9d-4c58-9453-24777c1a5bb0","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.242Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b7b3fafa-eb72-452d-b5d2-b46a5382db4f","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.248Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"b7b3fafa-eb72-452d-b5d2-b46a5382db4f","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.248Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"8bb7aaa5-b1fe-42f3-9ca9-61c142465411","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.253Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"8bb7aaa5-b1fe-42f3-9ca9-61c142465411","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.264Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"f046d876-ff23-49e5-b711-0583fa011d50","DeletionTimestamp":"2026-07-10 10:41:09 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:41:09.264Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"f046d876-ff23-49e5-b711-0583fa011d50","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:41:09.279Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c52b357e-9b5a-41f5-a7b9-ab923f915bad","DeletionTimestamp":"2026-07-10 10:41:09 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:41:09.279Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"c52b357e-9b5a-41f5-a7b9-ab923f915bad","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:41:09.292Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"a342babf-b7fb-4e0f-b660-61a521276153"}
{"level":"info","ts":"2026-07-10T10:41:09.292Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"38c7bba4-1dd6-46c3-aeb3-dac81e3011c2"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"9c8ab562-c0a2-404f-9e9e-f216289b796b"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"reconcileID":"84705f7f-6b61-4d24-a30d-8eb46ee46f72"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"113413a2-1dd2-4655-ba8f-cad31547a606"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","namespace":"test-ns-7lp9t","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3e1358cc-f5ae-4f01-b1c4-758ca0302426","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.305Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3e1358cc-f5ae-4f01-b1c4-758ca0302426","rayCluster":"fail-rkvnj"}
{"level":"error","ts":"2026-07-10T10:41:09.308Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3e1358cc-f5ae-4f01-b1c4-758ca0302426","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:41:09.308Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3e1358cc-f5ae-4f01-b1c4-758ca0302426","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.309Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3e1358cc-f5ae-4f01-b1c4-758ca0302426"}
{"level":"error","ts":"2026-07-10T10:41:09.309Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"3e1358cc-f5ae-4f01-b1c4-758ca0302426","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.311Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","name":"test-ns-7lp9t-fail-rkvnj","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:41:09.325Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8cab7b6-786f-4616-9216-956af398b090","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.325Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8cab7b6-786f-4616-9216-956af398b090","rayCluster":"fail-rkvnj"}
{"level":"error","ts":"2026-07-10T10:41:09.328Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8cab7b6-786f-4616-9216-956af398b090","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:41:09.328Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8cab7b6-786f-4616-9216-956af398b090","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.328Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8cab7b6-786f-4616-9216-956af398b090"}
{"level":"error","ts":"2026-07-10T10:41:09.328Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e8cab7b6-786f-4616-9216-956af398b090","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.331Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","serviceAccount":"fail-rkvnj-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:41:09.338Z","logger":"raycluster-resource","msg":"default","name":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.338Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-rkvnj","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:41:09.339Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e40d4eba-9dc2-4887-9ff7-7350497c7f27","rayCluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.339Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e40d4eba-9dc2-4887-9ff7-7350497c7f27","rayCluster":"fail-rkvnj"}
{"level":"error","ts":"2026-07-10T10:41:09.342Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e40d4eba-9dc2-4887-9ff7-7350497c7f27","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:41:09.342Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e40d4eba-9dc2-4887-9ff7-7350497c7f27","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.342Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e40d4eba-9dc2-4887-9ff7-7350497c7f27"}
{"level":"error","ts":"2026-07-10T10:41:09.342Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"e40d4eba-9dc2-4887-9ff7-7350497c7f27","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-rkvnj\" is forbidden: unable to create new content in namespace test-ns-7lp9t because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"46f75891-c162-4559-9b36-f6aa2961d2a8","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6","cluster":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6","namespace":"test-ns-7lp9t","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"8749b387-1fad-4eb6-bc87-889e0dfe3275"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"93a4a47e-5566-4a40-aa61-247ef14645f3","namespace":"test-ns-7lp9t","clusterName":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.349Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"8c9f9a77-df08-4722-9da9-c2fc9c72f8a6"}
{"level":"error","ts":"2026-07-10T10:41:09.353Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6","error":"rayclusters.ray.io \"fail-rkvnj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.353Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6"}
{"level":"error","ts":"2026-07-10T10:41:09.353Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"55ed4ef9-e3bb-4090-863e-d5e45ecfefb6","error":"rayclusters.ray.io \"fail-rkvnj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.353Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c928a060-2e27-486c-978b-20e79f24cf38","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:41:09.353Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c928a060-2e27-486c-978b-20e79f24cf38"}
{"level":"info","ts":"2026-07-10T10:41:09.353Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c928a060-2e27-486c-978b-20e79f24cf38","namespace":"test-ns-7lp9t","authClusters":0}
{"level":"info","ts":"2026-07-10T10:41:09.353Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c928a060-2e27-486c-978b-20e79f24cf38","namespace":"test-ns-7lp9t"}
{"level":"error","ts":"2026-07-10T10:41:09.358Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"93a4a47e-5566-4a40-aa61-247ef14645f3","issuer":"ray-selfsigned-issuer-fail-rkvnj","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-rkvnj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.358Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"93a4a47e-5566-4a40-aa61-247ef14645f3","errorCount":1}
{"level":"info","ts":"2026-07-10T10:41:09.358Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"93a4a47e-5566-4a40-aa61-247ef14645f3"}
{"level":"error","ts":"2026-07-10T10:41:09.358Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"93a4a47e-5566-4a40-aa61-247ef14645f3","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-rkvnj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:41:09.359Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"68f989b1-febc-464d-9975-fd6093431a27","namespacedName":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"}}
{"level":"info","ts":"2026-07-10T10:41:09.359Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"68f989b1-febc-464d-9975-fd6093431a27"}
{"level":"info","ts":"2026-07-10T10:41:09.359Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"68f989b1-febc-464d-9975-fd6093431a27","namespace":"test-ns-7lp9t","authClusters":0}
{"level":"info","ts":"2026-07-10T10:41:09.359Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"68f989b1-febc-464d-9975-fd6093431a27","namespace":"test-ns-7lp9t"}
{"level":"info","ts":"2026-07-10T10:41:09.363Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c82887a7-0ea1-42a9-985d-c9023176042d","namespace":"test-ns-7lp9t","clusterName":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.363Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-rkvnj","namespace":"test-ns-7lp9t"},"namespace":"test-ns-7lp9t","name":"fail-rkvnj","reconcileID":"c82887a7-0ea1-42a9-985d-c9023176042d","namespace":"test-ns-7lp9t","clusterName":"fail-rkvnj"}
{"level":"info","ts":"2026-07-10T10:41:09.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d692aa38-8c6f-4805-923d-7047732e05dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:09.927Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d692aa38-8c6f-4805-923d-7047732e05dc","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:09.927Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d692aa38-8c6f-4805-923d-7047732e05dc","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:09.927Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d692aa38-8c6f-4805-923d-7047732e05dc"}
{"level":"info","ts":"2026-07-10T10:41:09.927Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d692aa38-8c6f-4805-923d-7047732e05dc","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:10.360Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7a1babba-0120-4266-aabc-ed15743e1d65","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:41:10.360Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7a1babba-0120-4266-aabc-ed15743e1d65","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:41:10.360Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7a1babba-0120-4266-aabc-ed15743e1d65","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:41:10.360Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7a1babba-0120-4266-aabc-ed15743e1d65","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:41:12.279Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-7lp9t"},"reconcileID":"dceebdaf-00c9-49b5-95f4-09103ed71e7d"}
{"level":"info","ts":"2026-07-10T10:41:12.292Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-7lp9t"},"reconcileID":"34d9d4bb-bfb0-4b37-b72e-dc35005969ff"}
{"level":"info","ts":"2026-07-10T10:41:12.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c51e2532-97e9-4e89-9a87-6a4425068363","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:12.927Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c51e2532-97e9-4e89-9a87-6a4425068363","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:12.927Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c51e2532-97e9-4e89-9a87-6a4425068363","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:12.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c51e2532-97e9-4e89-9a87-6a4425068363"}
{"level":"info","ts":"2026-07-10T10:41:12.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c51e2532-97e9-4e89-9a87-6a4425068363","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:15.929Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"f7e35be3-49f8-4036-97ba-4610f4ef654b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:15.929Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"f7e35be3-49f8-4036-97ba-4610f4ef654b","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:15.929Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"f7e35be3-49f8-4036-97ba-4610f4ef654b","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:15.929Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"f7e35be3-49f8-4036-97ba-4610f4ef654b"}
{"level":"info","ts":"2026-07-10T10:41:15.929Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"f7e35be3-49f8-4036-97ba-4610f4ef654b","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:18.929Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ab6afd1-e865-4ce2-8a83-5fd803d2a6e9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:18.929Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ab6afd1-e865-4ce2-8a83-5fd803d2a6e9","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:18.929Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ab6afd1-e865-4ce2-8a83-5fd803d2a6e9","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:18.929Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ab6afd1-e865-4ce2-8a83-5fd803d2a6e9"}
{"level":"info","ts":"2026-07-10T10:41:18.929Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ab6afd1-e865-4ce2-8a83-5fd803d2a6e9","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:21.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6305fc1c-d955-4865-a0dd-0525bc18d7a5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:21.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6305fc1c-d955-4865-a0dd-0525bc18d7a5","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:21.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6305fc1c-d955-4865-a0dd-0525bc18d7a5","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:21.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6305fc1c-d955-4865-a0dd-0525bc18d7a5"}
{"level":"info","ts":"2026-07-10T10:41:21.930Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6305fc1c-d955-4865-a0dd-0525bc18d7a5","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:24.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"aa0c9b40-7d8b-4482-af6d-fb76a0fee325","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:24.931Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"aa0c9b40-7d8b-4482-af6d-fb76a0fee325","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:24.931Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"aa0c9b40-7d8b-4482-af6d-fb76a0fee325","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:24.931Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"aa0c9b40-7d8b-4482-af6d-fb76a0fee325"}
{"level":"info","ts":"2026-07-10T10:41:24.931Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"aa0c9b40-7d8b-4482-af6d-fb76a0fee325","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:27.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d3d651a3-7bb1-4e78-ad2b-cdbed3a564fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:27.931Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d3d651a3-7bb1-4e78-ad2b-cdbed3a564fc","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:27.931Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d3d651a3-7bb1-4e78-ad2b-cdbed3a564fc","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:27.932Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d3d651a3-7bb1-4e78-ad2b-cdbed3a564fc"}
{"level":"info","ts":"2026-07-10T10:41:27.932Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d3d651a3-7bb1-4e78-ad2b-cdbed3a564fc","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:28.126Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f020c86-167b-457e-a97f-0bfe6c0623f3","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:28.126Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f020c86-167b-457e-a97f-0bfe6c0623f3","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:41:28.126Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f020c86-167b-457e-a97f-0bfe6c0623f3","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:41:28.126Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"3f020c86-167b-457e-a97f-0bfe6c0623f3","rayCluster":"long-running-zpp4k","podIPs":["10.132.0.26","10.133.0.35"]}
{"level":"info","ts":"2026-07-10T10:41:30.933Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2dba16f2-3a07-4b1a-9a03-76043b2a0fdd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:30.933Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2dba16f2-3a07-4b1a-9a03-76043b2a0fdd","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:30.933Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2dba16f2-3a07-4b1a-9a03-76043b2a0fdd","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:30.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2dba16f2-3a07-4b1a-9a03-76043b2a0fdd"}
{"level":"info","ts":"2026-07-10T10:41:30.933Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2dba16f2-3a07-4b1a-9a03-76043b2a0fdd","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:33.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9eced25b-0ee3-49a9-8942-fabc2ecdd593","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:33.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9eced25b-0ee3-49a9-8942-fabc2ecdd593","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:33.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9eced25b-0ee3-49a9-8942-fabc2ecdd593","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:33.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9eced25b-0ee3-49a9-8942-fabc2ecdd593"}
{"level":"info","ts":"2026-07-10T10:41:33.934Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9eced25b-0ee3-49a9-8942-fabc2ecdd593","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:36.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b32aad76-0a0b-48b4-9731-9836a1c2848f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:36.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b32aad76-0a0b-48b4-9731-9836a1c2848f","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:36.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b32aad76-0a0b-48b4-9731-9836a1c2848f","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:36.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b32aad76-0a0b-48b4-9731-9836a1c2848f"}
{"level":"info","ts":"2026-07-10T10:41:36.934Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b32aad76-0a0b-48b4-9731-9836a1c2848f","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:39.935Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c0553306-9452-4a61-ac6b-d8e2b3059822","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:39.935Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c0553306-9452-4a61-ac6b-d8e2b3059822","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:39.935Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c0553306-9452-4a61-ac6b-d8e2b3059822","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:39.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c0553306-9452-4a61-ac6b-d8e2b3059822"}
{"level":"info","ts":"2026-07-10T10:41:39.936Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c0553306-9452-4a61-ac6b-d8e2b3059822","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:42.937Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7f11009f-4dd2-4310-b371-29eb6f10a51d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:42.937Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7f11009f-4dd2-4310-b371-29eb6f10a51d","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:42.937Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7f11009f-4dd2-4310-b371-29eb6f10a51d","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:42.937Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7f11009f-4dd2-4310-b371-29eb6f10a51d"}
{"level":"info","ts":"2026-07-10T10:41:42.937Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7f11009f-4dd2-4310-b371-29eb6f10a51d","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:45.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7e97a063-70f1-442f-a33c-8a1a8a9a63a9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:45.938Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7e97a063-70f1-442f-a33c-8a1a8a9a63a9","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:45.938Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7e97a063-70f1-442f-a33c-8a1a8a9a63a9","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:45.938Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7e97a063-70f1-442f-a33c-8a1a8a9a63a9"}
{"level":"info","ts":"2026-07-10T10:41:45.938Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"7e97a063-70f1-442f-a33c-8a1a8a9a63a9","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:48.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"45dc1089-a839-4bbe-b003-d6ebb49f9f3b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:48.938Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"45dc1089-a839-4bbe-b003-d6ebb49f9f3b","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:48.938Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"45dc1089-a839-4bbe-b003-d6ebb49f9f3b","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:48.938Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"45dc1089-a839-4bbe-b003-d6ebb49f9f3b"}
{"level":"info","ts":"2026-07-10T10:41:48.938Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"45dc1089-a839-4bbe-b003-d6ebb49f9f3b","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:51.939Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c3bc9027-c7f7-4f3d-8145-4f0f7fe4e4de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:51.939Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c3bc9027-c7f7-4f3d-8145-4f0f7fe4e4de","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:51.939Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c3bc9027-c7f7-4f3d-8145-4f0f7fe4e4de","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:51.940Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c3bc9027-c7f7-4f3d-8145-4f0f7fe4e4de"}
{"level":"info","ts":"2026-07-10T10:41:51.940Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c3bc9027-c7f7-4f3d-8145-4f0f7fe4e4de","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:54.940Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"57a4b0c6-a99c-4f13-ac60-4d05df062558","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:54.940Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"57a4b0c6-a99c-4f13-ac60-4d05df062558","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:54.940Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"57a4b0c6-a99c-4f13-ac60-4d05df062558","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:54.940Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"57a4b0c6-a99c-4f13-ac60-4d05df062558"}
{"level":"info","ts":"2026-07-10T10:41:54.940Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"57a4b0c6-a99c-4f13-ac60-4d05df062558","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:41:57.941Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a4f0b726-0dbd-44b3-bf0a-3ceb9a2a90e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:41:57.941Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a4f0b726-0dbd-44b3-bf0a-3ceb9a2a90e3","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:41:57.941Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a4f0b726-0dbd-44b3-bf0a-3ceb9a2a90e3","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:41:57.942Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a4f0b726-0dbd-44b3-bf0a-3ceb9a2a90e3"}
{"level":"info","ts":"2026-07-10T10:41:57.942Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a4f0b726-0dbd-44b3-bf0a-3ceb9a2a90e3","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:00.942Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9de54a3a-59cd-40be-9f99-b84388f15232","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:00.942Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9de54a3a-59cd-40be-9f99-b84388f15232","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:00.942Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9de54a3a-59cd-40be-9f99-b84388f15232","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:00.943Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9de54a3a-59cd-40be-9f99-b84388f15232"}
{"level":"info","ts":"2026-07-10T10:42:00.943Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9de54a3a-59cd-40be-9f99-b84388f15232","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:03.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"724a4577-36b1-474c-bf7f-13911953613c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:03.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"724a4577-36b1-474c-bf7f-13911953613c","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:03.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"724a4577-36b1-474c-bf7f-13911953613c","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:03.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"724a4577-36b1-474c-bf7f-13911953613c"}
{"level":"info","ts":"2026-07-10T10:42:03.944Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"724a4577-36b1-474c-bf7f-13911953613c","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:06.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"42701f8e-1eb5-4923-b684-f576be6cf47f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:06.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"42701f8e-1eb5-4923-b684-f576be6cf47f","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:06.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"42701f8e-1eb5-4923-b684-f576be6cf47f","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:06.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"42701f8e-1eb5-4923-b684-f576be6cf47f"}
{"level":"info","ts":"2026-07-10T10:42:06.944Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"42701f8e-1eb5-4923-b684-f576be6cf47f","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:09.945Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"272c9fac-16ee-4343-be65-b3fc39c6e0eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:09.945Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"272c9fac-16ee-4343-be65-b3fc39c6e0eb","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:09.945Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"272c9fac-16ee-4343-be65-b3fc39c6e0eb","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:09.946Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"272c9fac-16ee-4343-be65-b3fc39c6e0eb"}
{"level":"info","ts":"2026-07-10T10:42:09.946Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"272c9fac-16ee-4343-be65-b3fc39c6e0eb","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:10.361Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7727983f-b6d0-4ed1-8c2f-11bab244c93c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:42:10.361Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7727983f-b6d0-4ed1-8c2f-11bab244c93c","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:42:10.361Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7727983f-b6d0-4ed1-8c2f-11bab244c93c","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:42:10.361Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"7727983f-b6d0-4ed1-8c2f-11bab244c93c","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:42:12.947Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"27ccd3da-0c65-492d-b867-bf1cb3f6b191","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:12.947Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"27ccd3da-0c65-492d-b867-bf1cb3f6b191","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:12.947Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"27ccd3da-0c65-492d-b867-bf1cb3f6b191","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:12.947Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"27ccd3da-0c65-492d-b867-bf1cb3f6b191"}
{"level":"info","ts":"2026-07-10T10:42:12.947Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"27ccd3da-0c65-492d-b867-bf1cb3f6b191","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:15.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3a1c1834-0c8f-49a8-b4a2-1b9a008a267c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:15.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3a1c1834-0c8f-49a8-b4a2-1b9a008a267c","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:15.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3a1c1834-0c8f-49a8-b4a2-1b9a008a267c","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:15.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3a1c1834-0c8f-49a8-b4a2-1b9a008a267c"}
{"level":"info","ts":"2026-07-10T10:42:15.948Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3a1c1834-0c8f-49a8-b4a2-1b9a008a267c","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:18.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"207a3aea-ec7f-4130-8c59-68a392d936e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:18.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"207a3aea-ec7f-4130-8c59-68a392d936e4","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:18.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"207a3aea-ec7f-4130-8c59-68a392d936e4","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:18.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"207a3aea-ec7f-4130-8c59-68a392d936e4"}
{"level":"info","ts":"2026-07-10T10:42:18.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"207a3aea-ec7f-4130-8c59-68a392d936e4","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:21.950Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0a075c9d-459a-42af-830d-4bb793c50956","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:21.951Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0a075c9d-459a-42af-830d-4bb793c50956","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:21.951Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0a075c9d-459a-42af-830d-4bb793c50956","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:21.951Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0a075c9d-459a-42af-830d-4bb793c50956"}
{"level":"info","ts":"2026-07-10T10:42:21.951Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0a075c9d-459a-42af-830d-4bb793c50956","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","podName":"long-running-zpp4k-head-kcpg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:22.143Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e29f23f5-cf03-44a8-9110-2754785205e6","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b"}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b"}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:42:23.153Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"2531da29-aa79-4886-aec8-75790df4052b","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:41:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"45d8e398-3e96-4303-8b7c-e8f3c87c8aa5","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"23e5fe16-fe9a-4f36-a96f-b9c9b533dfcb","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1c0cba4f-9188-4f8c-8c1f-3d62b6767a0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1c0cba4f-9188-4f8c-8c1f-3d62b6767a0b","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1c0cba4f-9188-4f8c-8c1f-3d62b6767a0b","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"45d8e398-3e96-4303-8b7c-e8f3c87c8aa5","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"45d8e398-3e96-4303-8b7c-e8f3c87c8aa5","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1c0cba4f-9188-4f8c-8c1f-3d62b6767a0b"}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"45d8e398-3e96-4303-8b7c-e8f3c87c8aa5","rayCluster":"long-running-zpp4k","podIPs":["10.133.0.35","10.132.0.26"]}
{"level":"info","ts":"2026-07-10T10:42:23.167Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1c0cba4f-9188-4f8c-8c1f-3d62b6767a0b","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:23.184Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"23e5fe16-fe9a-4f36-a96f-b9c9b533dfcb","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:42:23.197Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"23e5fe16-fe9a-4f36-a96f-b9c9b533dfcb","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:42:23.197Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"23e5fe16-fe9a-4f36-a96f-b9c9b533dfcb","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:24.952Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4666e5a9-3a8d-493f-9fba-32bdddafef2e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:24.952Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4666e5a9-3a8d-493f-9fba-32bdddafef2e","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:24.952Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4666e5a9-3a8d-493f-9fba-32bdddafef2e","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:24.952Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4666e5a9-3a8d-493f-9fba-32bdddafef2e"}
{"level":"info","ts":"2026-07-10T10:42:24.952Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4666e5a9-3a8d-493f-9fba-32bdddafef2e","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525"}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525"}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:25.167Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:25.168Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:25.168Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"6fb27166-efee-4317-9861-23660bfeb525","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:27.618Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b"}
{"level":"info","ts":"2026-07-10T10:42:27.618Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b"}
{"level":"info","ts":"2026-07-10T10:42:27.618Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:27.619Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"06d2d334-5e4f-4754-99f5-614d55469f6b","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:27.953Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a58c2415-7778-4718-a794-dae4da8def4a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:27.953Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a58c2415-7778-4718-a794-dae4da8def4a","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:27.953Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a58c2415-7778-4718-a794-dae4da8def4a","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:27.953Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a58c2415-7778-4718-a794-dae4da8def4a"}
{"level":"info","ts":"2026-07-10T10:42:27.953Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a58c2415-7778-4718-a794-dae4da8def4a","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:28.127Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9350b136-3a52-475b-84db-17f06fb05ff1","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:28.127Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9350b136-3a52-475b-84db-17f06fb05ff1","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:42:28.127Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9350b136-3a52-475b-84db-17f06fb05ff1","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:42:28.127Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"9350b136-3a52-475b-84db-17f06fb05ff1","rayCluster":"long-running-zpp4k","podIPs":["10.133.0.35","10.132.0.26"]}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b"}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b"}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:28.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:28.623Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:42:28.623Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0fb5e9ae-6d6c-4106-b020-84cf2152501b","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"bf9ebd22-b77c-4d6b-9615-4c0af3771926","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ed78822e-f8d0-4c3b-90d0-c16babaa67e4","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"bf9ebd22-b77c-4d6b-9615-4c0af3771926","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"bf9ebd22-b77c-4d6b-9615-4c0af3771926","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"820168f2-805c-4be2-8322-02fc35b470a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"820168f2-805c-4be2-8322-02fc35b470a8","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"bf9ebd22-b77c-4d6b-9615-4c0af3771926","rayCluster":"long-running-zpp4k","podIPs":["10.133.0.35","10.132.0.26"]}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"820168f2-805c-4be2-8322-02fc35b470a8","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"820168f2-805c-4be2-8322-02fc35b470a8"}
{"level":"info","ts":"2026-07-10T10:42:28.635Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"820168f2-805c-4be2-8322-02fc35b470a8","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:28.650Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ed78822e-f8d0-4c3b-90d0-c16babaa67e4","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:42:28.664Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ed78822e-f8d0-4c3b-90d0-c16babaa67e4","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:42:28.664Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ed78822e-f8d0-4c3b-90d0-c16babaa67e4","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:30.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d"}
{"level":"info","ts":"2026-07-10T10:42:30.635Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d"}
{"level":"info","ts":"2026-07-10T10:42:30.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:30.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:30.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:30.635Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","podName":"long-running-zpp4k-head-kcpg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:30.636Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"ad8de652-3731-4b0c-8944-3486ef142a2d","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:30.954Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fd36e15e-7fd1-4a32-b482-295d87575f9d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:30.954Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fd36e15e-7fd1-4a32-b482-295d87575f9d","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:30.954Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fd36e15e-7fd1-4a32-b482-295d87575f9d","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:30.954Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fd36e15e-7fd1-4a32-b482-295d87575f9d"}
{"level":"info","ts":"2026-07-10T10:42:30.954Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fd36e15e-7fd1-4a32-b482-295d87575f9d","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:33.955Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ca76a12-7593-43c9-9177-4f9db0466075","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:33.955Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ca76a12-7593-43c9-9177-4f9db0466075","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:33.955Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ca76a12-7593-43c9-9177-4f9db0466075","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:33.955Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ca76a12-7593-43c9-9177-4f9db0466075"}
{"level":"info","ts":"2026-07-10T10:42:33.955Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4ca76a12-7593-43c9-9177-4f9db0466075","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:34.180Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723"}
{"level":"info","ts":"2026-07-10T10:42:34.180Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723"}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","podName":"long-running-zpp4k-small-group-worker-l4bv9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","podName":"long-running-zpp4k-small-group-worker-l4bv9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:42:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:42:34.181Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"0f1bbfa4-103a-4657-a79b-d8e14d6e9723","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:42:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1e40c1e9-6c67-42d6-8208-37242f61df26","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"633cab71-b2d0-4338-ac2a-71b508923f27","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3cf0d37c-d01d-4787-8f76-b60565f44eef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3cf0d37c-d01d-4787-8f76-b60565f44eef","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3cf0d37c-d01d-4787-8f76-b60565f44eef","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"633cab71-b2d0-4338-ac2a-71b508923f27","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"633cab71-b2d0-4338-ac2a-71b508923f27","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"633cab71-b2d0-4338-ac2a-71b508923f27","rayCluster":"long-running-zpp4k","podIPs":["10.133.0.35","10.132.0.26"]}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3cf0d37c-d01d-4787-8f76-b60565f44eef"}
{"level":"info","ts":"2026-07-10T10:42:34.194Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3cf0d37c-d01d-4787-8f76-b60565f44eef","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:34.206Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1e40c1e9-6c67-42d6-8208-37242f61df26","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:42:34.220Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1e40c1e9-6c67-42d6-8208-37242f61df26","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:42:34.220Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1e40c1e9-6c67-42d6-8208-37242f61df26","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707"}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707"}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","podName":"long-running-zpp4k-small-group-worker-l4bv9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","podName":"long-running-zpp4k-small-group-worker-l4bv9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:41:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:36.195Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"c756c1aa-2eb6-4a0c-bb1c-dd1aa54c1707","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:36.956Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e8521d22-a217-4932-b47e-2f9c1f4b72f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:36.956Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e8521d22-a217-4932-b47e-2f9c1f4b72f3","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:36.956Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e8521d22-a217-4932-b47e-2f9c1f4b72f3","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:36.956Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e8521d22-a217-4932-b47e-2f9c1f4b72f3"}
{"level":"info","ts":"2026-07-10T10:42:36.956Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e8521d22-a217-4932-b47e-2f9c1f4b72f3","RayCluster":"long-running-zpp4k","State":""}
{"level":"info","ts":"2026-07-10T10:42:39.668Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be"}
{"level":"info","ts":"2026-07-10T10:42:39.668Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be"}
{"level":"info","ts":"2026-07-10T10:42:39.668Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","1 head service found":"long-running-zpp4k-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:39.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","Found 1 head Pod":"long-running-zpp4k-head-kcpg5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:39.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","head Pod":"long-running-zpp4k-head-kcpg5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-zpp4k-head-kcpg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:39.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:39.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","worker Pod":"long-running-zpp4k-small-group-worker-l4bv9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-zpp4k-small-group-worker-l4bv9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:39.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:39.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:39.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:39.669Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:42:39.669Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"f3983ec4-d83f-43c9-9070-a1625be791be","name":"long-running-zpp4k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:42:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:39Z","stateTransitionTimes":{"ready":"2026-07-10T10:42:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.35","serviceIP":"10.133.0.35","podName":"long-running-zpp4k-head-kcpg5","serviceName":"long-running-zpp4k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:42:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:42:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:40:57Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:40:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6b84795a-4d5f-4f89-a990-2a03d5ea59c4","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"4f96e55e-ada7-4f49-adfe-781ca17fec29","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6b84795a-4d5f-4f89-a990-2a03d5ea59c4","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6b84795a-4d5f-4f89-a990-2a03d5ea59c4","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6b84795a-4d5f-4f89-a990-2a03d5ea59c4","rayCluster":"long-running-zpp4k","podIPs":["10.133.0.35","10.132.0.26"]}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","head service name":"long-running-zpp4k-head-svc","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","head service URL":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1"}
{"level":"info","ts":"2026-07-10T10:42:39.684Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","command":["if","!","ray","job","status","--address","http://long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","long-running-ssvzf",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","--no-wait","--submission-id","long-running-ssvzf","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","--follow","long-running-ssvzf"]}
{"level":"info","ts":"2026-07-10T10:42:39.696Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-07-10T10:42:39.696Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","SubmissionMode":"K8sJobMode","RayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.696Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","dashboardURL":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:42:39.696Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"22680d48-67db-4158-a929-81426b329ae1","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:42:39.702Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"4f96e55e-ada7-4f49-adfe-781ca17fec29","name":"long-running-zpp4k-head"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","head service name":"long-running-zpp4k-head-svc","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","head service URL":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","SubmissionMode":"K8sJobMode","RayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","dashboardURL":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:42:39.710Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:42:39.715Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"4f96e55e-ada7-4f49-adfe-781ca17fec29","name":"long-running-zpp4k-workers"}
{"level":"info","ts":"2026-07-10T10:42:39.715Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"4f96e55e-ada7-4f49-adfe-781ca17fec29","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.717Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:42:39.717Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef"}
{"level":"error","ts":"2026-07-10T10:42:39.717Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"eea91e8a-4f4e-4aa6-98fc-f1b61df6cbef","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.717Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"cce59a63-f8c4-47e1-a351-3dc9d3d0d528","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.717Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"cce59a63-f8c4-47e1-a351-3dc9d3d0d528","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.717Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"cce59a63-f8c4-47e1-a351-3dc9d3d0d528","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.717Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"cce59a63-f8c4-47e1-a351-3dc9d3d0d528"}
{"level":"error","ts":"2026-07-10T10:42:39.730Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"cce59a63-f8c4-47e1-a351-3dc9d3d0d528","JobId":"long-running-ssvzf","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.730Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"cce59a63-f8c4-47e1-a351-3dc9d3d0d528"}
{"level":"error","ts":"2026-07-10T10:42:39.730Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"cce59a63-f8c4-47e1-a351-3dc9d3d0d528","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.730Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9981756d-543d-47fc-9517-23625baff880","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.730Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9981756d-543d-47fc-9517-23625baff880","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.730Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9981756d-543d-47fc-9517-23625baff880","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.730Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9981756d-543d-47fc-9517-23625baff880"}
{"level":"error","ts":"2026-07-10T10:42:39.735Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9981756d-543d-47fc-9517-23625baff880","JobId":"long-running-ssvzf","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.735Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9981756d-543d-47fc-9517-23625baff880"}
{"level":"error","ts":"2026-07-10T10:42:39.735Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"9981756d-543d-47fc-9517-23625baff880","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.735Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8e49af2e-b022-4f3b-99ec-800ad6d3f4c4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.735Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8e49af2e-b022-4f3b-99ec-800ad6d3f4c4","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.735Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8e49af2e-b022-4f3b-99ec-800ad6d3f4c4","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.736Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8e49af2e-b022-4f3b-99ec-800ad6d3f4c4"}
{"level":"error","ts":"2026-07-10T10:42:39.740Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8e49af2e-b022-4f3b-99ec-800ad6d3f4c4","JobId":"long-running-ssvzf","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.741Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8e49af2e-b022-4f3b-99ec-800ad6d3f4c4"}
{"level":"error","ts":"2026-07-10T10:42:39.741Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8e49af2e-b022-4f3b-99ec-800ad6d3f4c4","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.741Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"de2b5ff6-7524-40fb-9d51-7de57c79df27","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.741Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"de2b5ff6-7524-40fb-9d51-7de57c79df27","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.741Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"de2b5ff6-7524-40fb-9d51-7de57c79df27","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.741Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"de2b5ff6-7524-40fb-9d51-7de57c79df27"}
{"level":"error","ts":"2026-07-10T10:42:39.745Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"de2b5ff6-7524-40fb-9d51-7de57c79df27","JobId":"long-running-ssvzf","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.745Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"de2b5ff6-7524-40fb-9d51-7de57c79df27"}
{"level":"error","ts":"2026-07-10T10:42:39.745Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"de2b5ff6-7524-40fb-9d51-7de57c79df27","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e4e749bc-c8e6-40ad-a103-22629353ed61","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e4e749bc-c8e6-40ad-a103-22629353ed61","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e4e749bc-c8e6-40ad-a103-22629353ed61","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e4e749bc-c8e6-40ad-a103-22629353ed61"}
{"level":"error","ts":"2026-07-10T10:42:39.828Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e4e749bc-c8e6-40ad-a103-22629353ed61","JobId":"long-running-ssvzf","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.828Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e4e749bc-c8e6-40ad-a103-22629353ed61"}
{"level":"error","ts":"2026-07-10T10:42:39.828Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e4e749bc-c8e6-40ad-a103-22629353ed61","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.989Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c080d132-e372-487d-ad0b-41ece9765a2c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:39.989Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c080d132-e372-487d-ad0b-41ece9765a2c","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:39.989Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c080d132-e372-487d-ad0b-41ece9765a2c","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:39.989Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c080d132-e372-487d-ad0b-41ece9765a2c"}
{"level":"error","ts":"2026-07-10T10:42:39.994Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c080d132-e372-487d-ad0b-41ece9765a2c","JobId":"long-running-ssvzf","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:39.994Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c080d132-e372-487d-ad0b-41ece9765a2c"}
{"level":"error","ts":"2026-07-10T10:42:39.994Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c080d132-e372-487d-ad0b-41ece9765a2c","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:40.315Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2d14b233-4f0d-4151-9b44-91de0e56d3e8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:40.315Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2d14b233-4f0d-4151-9b44-91de0e56d3e8","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2d14b233-4f0d-4151-9b44-91de0e56d3e8","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2d14b233-4f0d-4151-9b44-91de0e56d3e8"}
{"level":"error","ts":"2026-07-10T10:42:40.320Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2d14b233-4f0d-4151-9b44-91de0e56d3e8","JobId":"long-running-ssvzf","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:40.320Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2d14b233-4f0d-4151-9b44-91de0e56d3e8"}
{"level":"error","ts":"2026-07-10T10:42:40.320Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2d14b233-4f0d-4151-9b44-91de0e56d3e8","error":"Job long-running-ssvzf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:40.436Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d81bc1f0-7a65-403c-898b-02c2b1493451","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:40.436Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d81bc1f0-7a65-403c-898b-02c2b1493451","oldStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:42:40.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d81bc1f0-7a65-403c-898b-02c2b1493451","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","dashboardURL":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","dashboardURL":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:42:40.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"d81bc1f0-7a65-403c-898b-02c2b1493451","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-07-10T10:42:40.456Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"50edd454-f463-4105-b7ff-b8ce39e0fcfa","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:40.475Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"50edd454-f463-4105-b7ff-b8ce39e0fcfa","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.475Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"50edd454-f463-4105-b7ff-b8ce39e0fcfa","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"61153316-d593-4c5d-9b21-8ff226ff1b53"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"fd002b8b-329c-421e-8af4-ebe8a3686379","rayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"reconcileID":"e168697c-5cd3-471d-9380-9b7298bc03d0"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","namespace":"test-ns-pqpm4","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"fd002b8b-329c-421e-8af4-ebe8a3686379","pod":"long-running-zpp4k-head-kcpg5","ip":"10.133.0.35"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"fd002b8b-329c-421e-8af4-ebe8a3686379","pod":"long-running-zpp4k-small-group-worker-l4bv9","ip":"10.132.0.26"}
{"level":"info","ts":"2026-07-10T10:42:40.476Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"fd002b8b-329c-421e-8af4-ebe8a3686379","rayCluster":"long-running-zpp4k","podIPs":["10.133.0.35","10.132.0.26"]}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","name":"kuberay-gateway-access","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"The associated submitter Kubernetes Job for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"50edd454-f463-4105-b7ff-b8ce39e0fcfa","Submitter K8s Job":"long-running"}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"50edd454-f463-4105-b7ff-b8ce39e0fcfa","isJobDeleted":false}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"50edd454-f463-4105-b7ff-b8ce39e0fcfa"}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8b44c7f1-714c-4d2f-b780-96ccc62b569e","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8b44c7f1-714c-4d2f-b780-96ccc62b569e","RayCluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8b44c7f1-714c-4d2f-b780-96ccc62b569e","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8b44c7f1-714c-4d2f-b780-96ccc62b569e","Kubernetes Job":""}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8b44c7f1-714c-4d2f-b780-96ccc62b569e","isJobDeleted":true}
{"level":"info","ts":"2026-07-10T10:42:40.487Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"8b44c7f1-714c-4d2f-b780-96ccc62b569e"}
{"level":"info","ts":"2026-07-10T10:42:40.493Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","name":"test-ns-pqpm4-long-running-zpp4k","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:42:40.500Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","configMap":"kube-rbac-proxy-config-long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.507Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","serviceAccount":"long-running-zpp4k-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:42:40.529Z","logger":"raycluster-resource","msg":"default","name":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.529Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ae8b006b-f88b-4dd6-9ca2-a8766695acd2","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db","cluster":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db","namespace":"test-ns-pqpm4","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"fcef2c36-7529-47c7-83a8-312585ba8a1c"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ee71126d-0dc6-468a-9471-98f859e40a90","namespace":"test-ns-pqpm4","clusterName":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1efc2c6d-b0dd-49e4-870e-8920dee607e4","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1efc2c6d-b0dd-49e4-870e-8920dee607e4","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1efc2c6d-b0dd-49e4-870e-8920dee607e4","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1efc2c6d-b0dd-49e4-870e-8920dee607e4","Kubernetes Job":""}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1efc2c6d-b0dd-49e4-870e-8920dee607e4","isJobDeleted":true}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1efc2c6d-b0dd-49e4-870e-8920dee607e4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ssvzf","rayClusterName":"long-running-zpp4k","dashboardURL":"long-running-zpp4k-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:42:40.540Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1efc2c6d-b0dd-49e4-870e-8920dee607e4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"error","ts":"2026-07-10T10:42:40.544Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db","error":"rayclusters.ray.io \"long-running-zpp4k\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:40.544Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db"}
{"level":"error","ts":"2026-07-10T10:42:40.544Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"1059df11-c61e-4d9d-9239-9a53632972db","error":"rayclusters.ray.io \"long-running-zpp4k\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:40.544Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6ae00f71-e24c-4792-be35-079eeb56c57d","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.544Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6ae00f71-e24c-4792-be35-079eeb56c57d"}
{"level":"info","ts":"2026-07-10T10:42:40.544Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6ae00f71-e24c-4792-be35-079eeb56c57d","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:42:40.544Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"6ae00f71-e24c-4792-be35-079eeb56c57d","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:40.547Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"cbba7393-e5c2-49af-9573-ff3fd695fc23"}
{"level":"info","ts":"2026-07-10T10:42:40.547Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"655fea28-691d-4295-9104-4ac7d289a42a"}
{"level":"info","ts":"2026-07-10T10:42:40.550Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ea896597-d1fb-41e2-b210-1e6ba355985c","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.550Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ea896597-d1fb-41e2-b210-1e6ba355985c"}
{"level":"info","ts":"2026-07-10T10:42:40.550Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ea896597-d1fb-41e2-b210-1e6ba355985c","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:42:40.550Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ea896597-d1fb-41e2-b210-1e6ba355985c","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:40.554Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"66069424-b023-4719-adfe-4f7eace968e7","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:40.625Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"e0a49171-2953-4ed8-8f95-ed8458a1739f","namespacedName":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:40.625Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"e0a49171-2953-4ed8-8f95-ed8458a1739f"}
{"level":"info","ts":"2026-07-10T10:42:40.625Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"e0a49171-2953-4ed8-8f95-ed8458a1739f","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:42:40.625Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"e0a49171-2953-4ed8-8f95-ed8458a1739f","namespace":"test-ns-pqpm4"}
{"level":"error","ts":"2026-07-10T10:42:40.629Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ee71126d-0dc6-468a-9471-98f859e40a90","certificate":"ray-ca-certificate-long-running-zpp4k","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-zpp4k\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:40.646Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ee71126d-0dc6-468a-9471-98f859e40a90","errorCount":1}
{"level":"info","ts":"2026-07-10T10:42:40.646Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ee71126d-0dc6-468a-9471-98f859e40a90"}
{"level":"error","ts":"2026-07-10T10:42:40.646Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"ee71126d-0dc6-468a-9471-98f859e40a90","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-zpp4k\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:40.652Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"bcef72bf-abf5-4ee2-a69b-37131eaff409","namespace":"test-ns-pqpm4","clusterName":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.652Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-zpp4k","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-zpp4k","reconcileID":"bcef72bf-abf5-4ee2-a69b-37131eaff409","namespace":"test-ns-pqpm4","clusterName":"long-running-zpp4k"}
{"level":"info","ts":"2026-07-10T10:42:40.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b90f4c7c-96dc-4f95-972f-d8ce7783b9f6","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:41.475Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b78f814a-2b92-45e4-9c81-def5ea2c9685","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:41.475Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b78f814a-2b92-45e4-9c81-def5ea2c9685"}
{"level":"info","ts":"2026-07-10T10:42:41.475Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b78f814a-2b92-45e4-9c81-def5ea2c9685","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:42:41.475Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b78f814a-2b92-45e4-9c81-def5ea2c9685","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-07-10T10:42:41.489Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c8e4ea49-622f-473b-b942-7c9acd8a7410","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:41.489Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c8e4ea49-622f-473b-b942-7c9acd8a7410"}
{"level":"info","ts":"2026-07-10T10:42:41.489Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c8e4ea49-622f-473b-b942-7c9acd8a7410","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:42:41.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c8e4ea49-622f-473b-b942-7c9acd8a7410","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-10T10:40:57Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cfn5q","rayClusterName":"long-running-bfn9m","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:42:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:42:41.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c8e4ea49-622f-473b-b942-7c9acd8a7410","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:42:41.504Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2908214b-4985-4685-a36c-b605f70f8ef2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:41.504Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2908214b-4985-4685-a36c-b605f70f8ef2","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.504Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2908214b-4985-4685-a36c-b605f70f8ef2","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.504Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2908214b-4985-4685-a36c-b605f70f8ef2","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.509Z","logger":"raycluster-resource","msg":"default","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.509Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2908214b-4985-4685-a36c-b605f70f8ef2","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2908214b-4985-4685-a36c-b605f70f8ef2"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2908214b-4985-4685-a36c-b605f70f8ef2","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3e24f748-e0e7-490a-a55e-3e6ff149090e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3e24f748-e0e7-490a-a55e-3e6ff149090e","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3e24f748-e0e7-490a-a55e-3e6ff149090e","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","mode":"IntegratedOAuth","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"18f3a742-9251-4a23-a2ec-f49ae66b09be","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3e24f748-e0e7-490a-a55e-3e6ff149090e"}
{"level":"info","ts":"2026-07-10T10:42:41.520Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3e24f748-e0e7-490a-a55e-3e6ff149090e","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:41.527Z","logger":"raycluster-resource","msg":"default","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.527Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","name":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:42:41.532Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","name":"long-running-bfn9m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:41.538Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"384fc499-4e26-48d5-87eb-c633b03acb85","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:41.538Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"384fc499-4e26-48d5-87eb-c633b03acb85","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.538Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"384fc499-4e26-48d5-87eb-c633b03acb85","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.538Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"384fc499-4e26-48d5-87eb-c633b03acb85"}
{"level":"info","ts":"2026-07-10T10:42:41.538Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"384fc499-4e26-48d5-87eb-c633b03acb85","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:41.542Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"18f3a742-9251-4a23-a2ec-f49ae66b09be","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.542Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"17e4c644-9a07-4e04-8652-5b1e16936f12","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.546Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","name":"long-running-bfn9m-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:42:41.547Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","name":"long-running-bfn9m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"long-running-bfn9m"}}
{"level":"info","ts":"2026-07-10T10:42:41.547Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02"}
{"level":"error","ts":"2026-07-10T10:42:41.547Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f722244e-97a4-43f7-9a8f-19d3ca92fa02","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358"}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358"}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:42:41.548Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","name":"long-running-bfn9m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:41.554Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.554Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.554Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","name":"kuberay-gateway-access","namespace":"test-ns-pqpm4","operation":"created"}
{"level":"info","ts":"2026-07-10T10:42:41.562Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"17e4c644-9a07-4e04-8652-5b1e16936f12","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358"}
{"level":"error","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"8a2ba3b1-46ed-4ae0-96af-61a317926358","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"942ab57c-506e-4115-bc12-c1572571acfd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"942ab57c-506e-4115-bc12-c1572571acfd","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"942ab57c-506e-4115-bc12-c1572571acfd","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","name":"long-running-bfn9m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"942ab57c-506e-4115-bc12-c1572571acfd"}
{"level":"info","ts":"2026-07-10T10:42:41.563Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"942ab57c-506e-4115-bc12-c1572571acfd","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:41.625Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","name":"test-ns-pqpm4-long-running-bfn9m","operation":"created"}
{"level":"info","ts":"2026-07-10T10:42:41.625Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:42:41.625Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.629Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"17e4c644-9a07-4e04-8652-5b1e16936f12","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:42:41.629Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"17e4c644-9a07-4e04-8652-5b1e16936f12","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.630Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a40a6064-bf01-4a06-a9bf-afcd06b6a271","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.631Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","name":"kube-rbac-proxy-config-long-running-bfn9m","operation":"created"}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","name":"long-running-bfn9m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"long-running-bfn9m"}}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9"}
{"level":"error","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"136cc4a3-3858-41d3-8e25-3e9343e742f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383"}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383"}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383"}
{"level":"error","ts":"2026-07-10T10:42:41.632Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"b55040e0-4164-4151-b7b3-9b6698be4383","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.636Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m"}
{"level":"error","ts":"2026-07-10T10:42:41.637Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.637Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086"}
{"level":"error","ts":"2026-07-10T10:42:41.637Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a4d3a427-1cb8-4b34-b28b-86a8eddd8086","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.637Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a64d9b35-fb9d-4a7f-a014-2d234d17dd1d","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.637Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a64d9b35-fb9d-4a7f-a014-2d234d17dd1d","mode":"IntegratedOAuth","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.637Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a64d9b35-fb9d-4a7f-a014-2d234d17dd1d","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed"}
{"level":"info","ts":"2026-07-10T10:42:41.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed"}
{"level":"info","ts":"2026-07-10T10:42:41.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:41.725Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a40a6064-bf01-4a06-a9bf-afcd06b6a271","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:42:41.725Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:42:41.725Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:41.726Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.726Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:41.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.726Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed"}
{"level":"error","ts":"2026-07-10T10:42:41.726Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"33697ed2-33ef-40a4-a6af-702bc291b6ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.737Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a64d9b35-fb9d-4a7f-a014-2d234d17dd1d","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.737Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c9e2d5d6-acd4-4ba4-8005-c0c31d2f0020","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:41.737Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c9e2d5d6-acd4-4ba4-8005-c0c31d2f0020","mode":"IntegratedOAuth","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.737Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c9e2d5d6-acd4-4ba4-8005-c0c31d2f0020","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","rayCluster":"long-running-bfn9m","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:42:41.825Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","error":"Certificate.cert-manager.io \"ray-head-cert-long-running-bfn9m\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2"}
{"level":"error","ts":"2026-07-10T10:42:41.825Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1f3943d1-9214-4b48-841f-17afa1bd65f2","error":"Certificate.cert-manager.io \"ray-head-cert-long-running-bfn9m\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a40a6064-bf01-4a06-a9bf-afcd06b6a271","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a40a6064-bf01-4a06-a9bf-afcd06b6a271","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"b684fd5a-ea8d-4a6f-92c1-0bd48c52b6db","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-07-10T10:42:41.825Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c9e2d5d6-acd4-4ba4-8005-c0c31d2f0020","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c9e2d5d6-acd4-4ba4-8005-c0c31d2f0020"}
{"level":"error","ts":"2026-07-10T10:42:41.825Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c9e2d5d6-acd4-4ba4-8005-c0c31d2f0020","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.826Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:41.826Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec"}
{"level":"error","ts":"2026-07-10T10:42:41.826Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0afebd39-540b-4351-ade5-14c93e8e7bec","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.826Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1288114d-dafd-4912-8d4a-864f77a87883","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:41.925Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"62afd4e2-a0ef-4e9e-8948-e55e3f9bfa20","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"62afd4e2-a0ef-4e9e-8948-e55e3f9bfa20","mode":"IntegratedOAuth","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"62afd4e2-a0ef-4e9e-8948-e55e3f9bfa20","cluster":"long-running-bfn9m"}
{"level":"error","ts":"2026-07-10T10:42:41.925Z","msg":"Failed to create Ray head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"b684fd5a-ea8d-4a6f-92c1-0bd48c52b6db","error":"certificates.cert-manager.io \"ray-head-cert-long-running-bfn9m\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayHeadCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:650\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:193\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:42:42.025Z","msg":"Failed to create Ray head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"b684fd5a-ea8d-4a6f-92c1-0bd48c52b6db","error":"certificates.cert-manager.io \"ray-head-cert-long-running-bfn9m\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:194\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:41.930Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1288114d-dafd-4912-8d4a-864f77a87883","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"b684fd5a-ea8d-4a6f-92c1-0bd48c52b6db"}
{"level":"info","ts":"2026-07-10T10:42:41.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29"}
{"level":"error","ts":"2026-07-10T10:42:42.025Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"b684fd5a-ea8d-4a6f-92c1-0bd48c52b6db","error":"certificates.cert-manager.io \"ray-head-cert-long-running-bfn9m\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29"}
{"level":"info","ts":"2026-07-10T10:42:41.925Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e25fd67a-2587-471e-af51-2996680ee949","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e25fd67a-2587-471e-af51-2996680ee949","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e25fd67a-2587-471e-af51-2996680ee949","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"35bf299e-01ed-45dc-84d4-b8ebba5ebf77","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e25fd67a-2587-471e-af51-2996680ee949"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e25fd67a-2587-471e-af51-2996680ee949","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"35bf299e-01ed-45dc-84d4-b8ebba5ebf77"}
{"level":"info","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29"}
{"level":"error","ts":"2026-07-10T10:42:42.025Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"242764d1-8116-4189-83e6-a3230e155b29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:42.125Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a14944b3-a129-4e9c-a538-af58baac7016","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:42.125Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a14944b3-a129-4e9c-a538-af58baac7016"}
{"level":"info","ts":"2026-07-10T10:42:42.126Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"62afd4e2-a0ef-4e9e-8948-e55e3f9bfa20","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:42.130Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1288114d-dafd-4912-8d4a-864f77a87883","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:42:42.130Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1288114d-dafd-4912-8d4a-864f77a87883","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78"}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78"}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","error":"head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78"}
{"level":"error","ts":"2026-07-10T10:42:42.346Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"db8d06ab-71e4-4e80-93d7-afa21ab72b78","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:42.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20"}
{"level":"info","ts":"2026-07-10T10:42:42.986Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20"}
{"level":"info","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","error":"worker certificate secret ray-worker-secret-long-running-bfn9m not found"}
{"level":"error","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-bfn9m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-bfn9m not found"}
{"level":"info","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"long-running-bfn9m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20"}
{"level":"error","ts":"2026-07-10T10:42:42.987Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"ea65b4e1-2a7d-41da-8035-0e16d8080a20","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-bfn9m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:43.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fc9fc377-9f81-4934-8551-2e905fcacb9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:43.961Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fc9fc377-9f81-4934-8551-2e905fcacb9a","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:43.961Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fc9fc377-9f81-4934-8551-2e905fcacb9a","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:43.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fc9fc377-9f81-4934-8551-2e905fcacb9a"}
{"level":"info","ts":"2026-07-10T10:42:43.961Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"fc9fc377-9f81-4934-8551-2e905fcacb9a","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","mode":"IntegratedOAuth","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","cluster":"long-running-bfn9m","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","cluster":"long-running-bfn9m","authType":"IntegratedOAuth","serviceAccount":"long-running-bfn9m-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-bfn9m","ray.io/group":"headgroup","ray.io/identifier":"long-running-bfn9m-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:42:44.267Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","enabled":true}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","name":"long-running-bfn9m-head-8zfts"}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da"}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","nodeType":"worker","rayStartParams":{"address":"long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","rayStartCmd":"ray start  --address=long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:42:44.284Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","enabled":true}
{"level":"info","ts":"2026-07-10T10:42:44.300Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","name":"long-running-bfn9m-small-group-worker-kzbpb"}
{"level":"info","ts":"2026-07-10T10:42:44.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.300Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-bfn9m-head-svc"},"newHead":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"}}
{"level":"info","ts":"2026-07-10T10:42:44.300Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"77d58fbf-4bd8-400e-934b-3dfe71cf29da","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9f20acab-23dd-4e59-b349-17f9a751b994","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a7c22f54-fe1b-4522-b761-ce3be960dae0","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"249f6cb4-f68a-45ab-b979-783582f3a89d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"249f6cb4-f68a-45ab-b979-783582f3a89d","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9f20acab-23dd-4e59-b349-17f9a751b994","rayCluster":"long-running-bfn9m","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"249f6cb4-f68a-45ab-b979-783582f3a89d","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"249f6cb4-f68a-45ab-b979-783582f3a89d"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.314Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"249f6cb4-f68a-45ab-b979-783582f3a89d","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:44.315Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-bfn9m-head-svc"},"newHead":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"}}
{"level":"info","ts":"2026-07-10T10:42:44.315Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","name":"long-running-bfn9m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"long-running-bfn9m"}}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939"}
{"level":"error","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"43bf5bc1-1f94-4a9b-8fbc-5e5bed74b939","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:44.324Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3af77976-2a25-40e9-88c6-a510383d1a33","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:44.327Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a7c22f54-fe1b-4522-b761-ce3be960dae0","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:42:44.329Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41"}
{"level":"info","ts":"2026-07-10T10:42:44.329Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41"}
{"level":"info","ts":"2026-07-10T10:42:44.329Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:44.330Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bbafbb92-7f61-4dca-8f84-b68117acad41","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:42:44.331Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"339b8338-a352-4d59-a3b1-72094f7b9723","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:44.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a7c22f54-fe1b-4522-b761-ce3be960dae0","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:42:44.341Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"a7c22f54-fe1b-4522-b761-ce3be960dae0","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7d814101-1195-40e1-8a1d-fdd4dbcb5e86","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9097de85-f7a5-45b5-977b-1075b2b31eb9","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4901429b-a4c9-4cce-8eeb-9b590b8bdd59","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4901429b-a4c9-4cce-8eeb-9b590b8bdd59","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4901429b-a4c9-4cce-8eeb-9b590b8bdd59","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9097de85-f7a5-45b5-977b-1075b2b31eb9","rayCluster":"long-running-bfn9m","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4901429b-a4c9-4cce-8eeb-9b590b8bdd59"}
{"level":"info","ts":"2026-07-10T10:42:44.343Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"4901429b-a4c9-4cce-8eeb-9b590b8bdd59","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:44.352Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","name":"long-running-bfn9m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"long-running-bfn9m"}}
{"level":"info","ts":"2026-07-10T10:42:44.352Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6"}
{"level":"error","ts":"2026-07-10T10:42:44.352Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"e9f57c8d-c894-4e3a-8af3-0c9fd43eccf6","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-bfn9m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:42:44.356Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7d814101-1195-40e1-8a1d-fdd4dbcb5e86","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:44.358Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"cf674115-3a27-442d-81dc-50b841aac9df","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:44.366Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7d814101-1195-40e1-8a1d-fdd4dbcb5e86","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:42:44.366Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7d814101-1195-40e1-8a1d-fdd4dbcb5e86","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.751Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.752Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:44.752Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"484a1266-5888-4a0f-b7f9-b4cdfd126fcf","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:44.754Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"76e8aa40-ebe4-4ab7-9068-1d8764bf6ffa","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:45.258Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9dcb69c7-0ace-4646-a1fc-6fc1b214e13c","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"newHead":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"}}
{"level":"info","ts":"2026-07-10T10:42:45.703Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"7f512a1f-01f1-40a6-a6e8-f0bc58cd4bfe","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:45.715Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"937acdfe-d02b-40fc-8764-4bd8ea08038d","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:45.715Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1b8fa79a-a8ba-446b-a546-4abe2c12608b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:45.715Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"78471563-a27a-4a0f-80ef-a05606076f6b","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:45.715Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1b8fa79a-a8ba-446b-a546-4abe2c12608b","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:45.715Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"937acdfe-d02b-40fc-8764-4bd8ea08038d","pod":"long-running-bfn9m-head-8zfts","ip":"10.132.0.28"}
{"level":"info","ts":"2026-07-10T10:42:45.715Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"937acdfe-d02b-40fc-8764-4bd8ea08038d","pod":"long-running-bfn9m-small-group-worker-kzbpb","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-10T10:42:45.715Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1b8fa79a-a8ba-446b-a546-4abe2c12608b","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:45.716Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1b8fa79a-a8ba-446b-a546-4abe2c12608b"}
{"level":"info","ts":"2026-07-10T10:42:45.716Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1b8fa79a-a8ba-446b-a546-4abe2c12608b","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:45.727Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"78471563-a27a-4a0f-80ef-a05606076f6b","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:42:45.728Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"937acdfe-d02b-40fc-8764-4bd8ea08038d","certificate":"ray-head-cert-long-running-bfn9m","dnsNames":["long-running-bfn9m-head-svc","localhost","long-running-bfn9m-head-svc.test-ns-pqpm4.svc","long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local"],"ipAddresses":["10.132.0.28","10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:42:45.738Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"937acdfe-d02b-40fc-8764-4bd8ea08038d","certificate":"ray-worker-cert-long-running-bfn9m","dnsNames":["long-running-bfn9m-worker-svc","localhost","long-running-bfn9m-small-group","long-running-bfn9m-worker-svc.test-ns-pqpm4.svc","long-running-bfn9m-worker-svc.test-ns-pqpm4.svc.cluster.local","*.long-running-bfn9m-worker-svc.test-ns-pqpm4.svc","*.long-running-bfn9m-worker-svc.test-ns-pqpm4.svc.cluster.local","*-worker-*.test-ns-pqpm4.svc","*-worker-*.test-ns-pqpm4.svc.cluster.local"],"ipAddresses":["10.132.0.28","10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:42:45.738Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"937acdfe-d02b-40fc-8764-4bd8ea08038d","rayCluster":"long-running-bfn9m","podIPs":["10.132.0.28","10.133.0.36"]}
{"level":"info","ts":"2026-07-10T10:42:45.740Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"78471563-a27a-4a0f-80ef-a05606076f6b","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:42:45.740Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"78471563-a27a-4a0f-80ef-a05606076f6b","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:46.963Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1d2fd273-1cde-4f2f-a99b-a1f5a4f9dd96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:46.963Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1d2fd273-1cde-4f2f-a99b-a1f5a4f9dd96","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:46.963Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1d2fd273-1cde-4f2f-a99b-a1f5a4f9dd96","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:46.963Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1d2fd273-1cde-4f2f-a99b-a1f5a4f9dd96"}
{"level":"info","ts":"2026-07-10T10:42:46.963Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"1d2fd273-1cde-4f2f-a99b-a1f5a4f9dd96","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:47.716Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bc587c73-11c8-4345-92ec-12bf5313f634","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:49.964Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e91c162e-3a0a-4608-b2b6-7c001663fd69","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:49.964Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e91c162e-3a0a-4608-b2b6-7c001663fd69","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:49.964Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e91c162e-3a0a-4608-b2b6-7c001663fd69","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:49.964Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e91c162e-3a0a-4608-b2b6-7c001663fd69"}
{"level":"info","ts":"2026-07-10T10:42:49.964Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e91c162e-3a0a-4608-b2b6-7c001663fd69","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:50.280Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"bf32418f-77bc-430b-a4a9-90a1d8b19a81","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:50.727Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"1c45b7b2-de92-471e-aaaf-9280e2c8b9fa","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","podName":"long-running-bfn9m-head-8zfts","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:51.288Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"d789177a-fdc7-4519-96af-ca75d249212e","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc"}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc"}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:42:51.734Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"f63bc698-3d3f-490d-8d2f-faf93b7781bc","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1c8db6d7-f7a6-4bdf-9bbe-d5b76461595f","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"87e145f6-96df-4141-bd7e-b3255eedded0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"87e145f6-96df-4141-bd7e-b3255eedded0","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"87e145f6-96df-4141-bd7e-b3255eedded0","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"55e1aaa8-466a-43ef-a81b-be26ccb00203","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"87e145f6-96df-4141-bd7e-b3255eedded0"}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"87e145f6-96df-4141-bd7e-b3255eedded0","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"55e1aaa8-466a-43ef-a81b-be26ccb00203","pod":"long-running-bfn9m-head-8zfts","ip":"10.132.0.28"}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"55e1aaa8-466a-43ef-a81b-be26ccb00203","pod":"long-running-bfn9m-small-group-worker-kzbpb","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-10T10:42:51.747Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"55e1aaa8-466a-43ef-a81b-be26ccb00203","rayCluster":"long-running-bfn9m","podIPs":["10.132.0.28","10.133.0.36"]}
{"level":"info","ts":"2026-07-10T10:42:51.758Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1c8db6d7-f7a6-4bdf-9bbe-d5b76461595f","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:42:51.769Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1c8db6d7-f7a6-4bdf-9bbe-d5b76461595f","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:42:51.769Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"1c8db6d7-f7a6-4bdf-9bbe-d5b76461595f","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:52.964Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e3a12b3a-f1e5-455f-9a94-782e850f12b5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:52.964Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e3a12b3a-f1e5-455f-9a94-782e850f12b5","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:52.964Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e3a12b3a-f1e5-455f-9a94-782e850f12b5","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:52.964Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e3a12b3a-f1e5-455f-9a94-782e850f12b5"}
{"level":"info","ts":"2026-07-10T10:42:52.964Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e3a12b3a-f1e5-455f-9a94-782e850f12b5","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae"}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae"}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:53.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:42:53.748Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:53.748Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"07ee55bd-c1ed-4954-ab48-dfcdfd43c1ae","seconds":300}
{"level":"info","ts":"2026-07-10T10:42:55.965Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b50747ce-3c84-44a4-9853-0eec477cb06c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:55.965Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b50747ce-3c84-44a4-9853-0eec477cb06c","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:55.965Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b50747ce-3c84-44a4-9853-0eec477cb06c","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:55.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b50747ce-3c84-44a4-9853-0eec477cb06c"}
{"level":"info","ts":"2026-07-10T10:42:55.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b50747ce-3c84-44a4-9853-0eec477cb06c","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:58.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3af3277c-1f64-49a6-a513-e0ef5d7b2530","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:42:58.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3af3277c-1f64-49a6-a513-e0ef5d7b2530","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:42:58.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3af3277c-1f64-49a6-a513-e0ef5d7b2530","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:42:58.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3af3277c-1f64-49a6-a513-e0ef5d7b2530"}
{"level":"info","ts":"2026-07-10T10:42:58.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3af3277c-1f64-49a6-a513-e0ef5d7b2530","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309"}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309"}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:42:59.925Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70f34714-0be0-4576-9e3f-e77b5633f309","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a"}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a"}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:00.327Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"9e772b16-0e81-4827-977e-945536cf229a","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:01.333Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b"}
{"level":"info","ts":"2026-07-10T10:43:01.333Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b"}
{"level":"info","ts":"2026-07-10T10:43:01.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:01.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","podName":"long-running-bfn9m-head-8zfts","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:43:01.334Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"380abaa6-0518-4294-95d7-640fb5c7556b","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:42:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:01.345Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7f5bcce9-f842-48d3-8ec7-1ec0ae5a497b","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:01.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"998bbdf5-7d0b-4a91-82a7-0172245ce5d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:01.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"998bbdf5-7d0b-4a91-82a7-0172245ce5d3","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:01.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"998bbdf5-7d0b-4a91-82a7-0172245ce5d3","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:01.345Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"e9ab9645-1114-476d-95b6-026055e8cd57","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:01.346Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"e9ab9645-1114-476d-95b6-026055e8cd57","pod":"long-running-bfn9m-head-8zfts","ip":"10.132.0.28"}
{"level":"info","ts":"2026-07-10T10:43:01.346Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"e9ab9645-1114-476d-95b6-026055e8cd57","pod":"long-running-bfn9m-small-group-worker-kzbpb","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-10T10:43:01.346Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"998bbdf5-7d0b-4a91-82a7-0172245ce5d3"}
{"level":"info","ts":"2026-07-10T10:43:01.346Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"998bbdf5-7d0b-4a91-82a7-0172245ce5d3","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:43:01.346Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"e9ab9645-1114-476d-95b6-026055e8cd57","rayCluster":"long-running-bfn9m","podIPs":["10.132.0.28","10.133.0.36"]}
{"level":"info","ts":"2026-07-10T10:43:01.363Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7f5bcce9-f842-48d3-8ec7-1ec0ae5a497b","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:43:01.372Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7f5bcce9-f842-48d3-8ec7-1ec0ae5a497b","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:43:01.372Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7f5bcce9-f842-48d3-8ec7-1ec0ae5a497b","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:01.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"36656a7e-c3d0-4aeb-bd61-3cf4ee99f960","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:01.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"36656a7e-c3d0-4aeb-bd61-3cf4ee99f960","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:01.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"36656a7e-c3d0-4aeb-bd61-3cf4ee99f960","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:01.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"36656a7e-c3d0-4aeb-bd61-3cf4ee99f960"}
{"level":"info","ts":"2026-07-10T10:43:01.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"36656a7e-c3d0-4aeb-bd61-3cf4ee99f960","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:43:02.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa"}
{"level":"info","ts":"2026-07-10T10:43:02.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa"}
{"level":"info","ts":"2026-07-10T10:43:02.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:02.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:43:02.767Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6238b975-b385-47e0-ba27-c0d2fe56c9fa","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"8ba0917e-1243-4a57-82c8-8ba5dc0a4c93","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7ce869c0-8b0c-44c2-9bc1-931d54772a32","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b3594ae1-cb28-4436-8bb4-1702eff769fa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b3594ae1-cb28-4436-8bb4-1702eff769fa","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b3594ae1-cb28-4436-8bb4-1702eff769fa","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7ce869c0-8b0c-44c2-9bc1-931d54772a32","pod":"long-running-bfn9m-head-8zfts","ip":"10.132.0.28"}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7ce869c0-8b0c-44c2-9bc1-931d54772a32","pod":"long-running-bfn9m-small-group-worker-kzbpb","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7ce869c0-8b0c-44c2-9bc1-931d54772a32","rayCluster":"long-running-bfn9m","podIPs":["10.132.0.28","10.133.0.36"]}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b3594ae1-cb28-4436-8bb4-1702eff769fa"}
{"level":"info","ts":"2026-07-10T10:43:02.779Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"b3594ae1-cb28-4436-8bb4-1702eff769fa","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:43:02.792Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"8ba0917e-1243-4a57-82c8-8ba5dc0a4c93","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:43:02.804Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"8ba0917e-1243-4a57-82c8-8ba5dc0a4c93","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:43:02.804Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"8ba0917e-1243-4a57-82c8-8ba5dc0a4c93","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83"}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83"}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","podName":"long-running-bfn9m-small-group-worker-kzbpb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:42:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:03.346Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"46d1d180-600e-4bd9-b25f-f10bbbb70a83","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:04.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"16ad4a23-bdc6-4f74-8c17-913a580005fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:04.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"16ad4a23-bdc6-4f74-8c17-913a580005fe","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:04.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"16ad4a23-bdc6-4f74-8c17-913a580005fe","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:04.968Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"16ad4a23-bdc6-4f74-8c17-913a580005fe"}
{"level":"info","ts":"2026-07-10T10:43:04.968Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"16ad4a23-bdc6-4f74-8c17-913a580005fe","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:43:07.968Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"66b553c1-38d9-429d-b159-2b085de8be1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:07.968Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"66b553c1-38d9-429d-b159-2b085de8be1d","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:07.968Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"66b553c1-38d9-429d-b159-2b085de8be1d","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:07.968Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"66b553c1-38d9-429d-b159-2b085de8be1d"}
{"level":"info","ts":"2026-07-10T10:43:07.968Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"66b553c1-38d9-429d-b159-2b085de8be1d","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:43:10.362Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"198905c2-2617-4044-bc44-e04315ca2fa2","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:43:10.362Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"198905c2-2617-4044-bc44-e04315ca2fa2","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:43:10.362Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"198905c2-2617-4044-bc44-e04315ca2fa2","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:43:10.362Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"198905c2-2617-4044-bc44-e04315ca2fa2","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:43:10.969Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5734b849-e5c9-468e-854b-c13d4e48e4dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:10.969Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5734b849-e5c9-468e-854b-c13d4e48e4dd","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:10.969Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5734b849-e5c9-468e-854b-c13d4e48e4dd","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:10.969Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5734b849-e5c9-468e-854b-c13d4e48e4dd"}
{"level":"info","ts":"2026-07-10T10:43:10.969Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"5734b849-e5c9-468e-854b-c13d4e48e4dd","RayCluster":"long-running-bfn9m","State":""}
{"level":"info","ts":"2026-07-10T10:43:12.126Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"fa501e18-d339-429c-9b69-c50ac4d39672","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.126Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"fa501e18-d339-429c-9b69-c50ac4d39672","pod":"long-running-bfn9m-head-8zfts","ip":"10.132.0.28"}
{"level":"info","ts":"2026-07-10T10:43:12.126Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"fa501e18-d339-429c-9b69-c50ac4d39672","pod":"long-running-bfn9m-small-group-worker-kzbpb","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-10T10:43:12.126Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"fa501e18-d339-429c-9b69-c50ac4d39672","rayCluster":"long-running-bfn9m","podIPs":["10.132.0.28","10.133.0.36"]}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8"}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8"}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","1 head service found":"long-running-bfn9m-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","Found 1 head Pod":"long-running-bfn9m-head-8zfts","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","head Pod":"long-running-bfn9m-head-8zfts","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-bfn9m-head-8zfts. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","worker Pod":"long-running-bfn9m-small-group-worker-kzbpb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-bfn9m-small-group-worker-kzbpb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:12.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:12.366Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:43:12.366Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"6dc5b507-e8e1-4def-8c6f-4e9d7e8d59f8","name":"long-running-bfn9m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:12Z","stateTransitionTimes":{"ready":"2026-07-10T10:43:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.28","serviceIP":"10.132.0.28","podName":"long-running-bfn9m-head-8zfts","serviceName":"long-running-bfn9m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:43:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:42:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:42:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:12.382Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"43eea950-73d0-42a1-89f5-f66bdb7f5c70","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.382Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"21891042-df40-4f63-a918-aa65d4f95094","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.382Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.382Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.382Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.382Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"21891042-df40-4f63-a918-aa65d4f95094","pod":"long-running-bfn9m-head-8zfts","ip":"10.132.0.28"}
{"level":"info","ts":"2026-07-10T10:43:12.382Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"21891042-df40-4f63-a918-aa65d4f95094","pod":"long-running-bfn9m-small-group-worker-kzbpb","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-10T10:43:12.383Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db"}
{"level":"info","ts":"2026-07-10T10:43:12.383Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"21891042-df40-4f63-a918-aa65d4f95094","rayCluster":"long-running-bfn9m","podIPs":["10.132.0.28","10.133.0.36"]}
{"level":"info","ts":"2026-07-10T10:43:12.383Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","head service name":"long-running-bfn9m-head-svc","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:12.383Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","head service URL":"long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:43:12.383Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db"}
{"level":"info","ts":"2026-07-10T10:43:12.383Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","command":["if","!","ray","job","status","--address","http://long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:8265","long-running-cfn5q",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:8265","--no-wait","--submission-id","long-running-cfn5q","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:8265","--follow","long-running-cfn5q"]}
{"level":"info","ts":"2026-07-10T10:43:12.398Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-07-10T10:43:12.399Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","SubmissionMode":"K8sJobMode","RayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.399Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cfn5q","rayClusterName":"long-running-bfn9m","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:42:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cfn5q","rayClusterName":"long-running-bfn9m","dashboardURL":"long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:42:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:12.399Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"04e1593f-8858-427f-85dd-e78c00c244db","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:43:12.407Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"43eea950-73d0-42a1-89f5-f66bdb7f5c70","name":"long-running-bfn9m-head"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","head service name":"long-running-bfn9m-head-svc","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","head service URL":"long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","SubmissionMode":"K8sJobMode","RayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cfn5q","rayClusterName":"long-running-bfn9m","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:42:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cfn5q","rayClusterName":"long-running-bfn9m","dashboardURL":"long-running-bfn9m-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:42:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:12.416Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:43:12.423Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"43eea950-73d0-42a1-89f5-f66bdb7f5c70","name":"long-running-bfn9m-workers"}
{"level":"info","ts":"2026-07-10T10:43:12.423Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"43eea950-73d0-42a1-89f5-f66bdb7f5c70","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.424Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:43:12.424Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f"}
{"level":"error","ts":"2026-07-10T10:43:12.424Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"c59e98c0-5e24-4fe9-8b8a-61c3b85d7e7f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.424Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e61373b1-9c9a-4de0-9eaa-05f11eb78b44","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.424Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e61373b1-9c9a-4de0-9eaa-05f11eb78b44","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.424Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e61373b1-9c9a-4de0-9eaa-05f11eb78b44","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.424Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e61373b1-9c9a-4de0-9eaa-05f11eb78b44"}
{"level":"error","ts":"2026-07-10T10:43:12.435Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e61373b1-9c9a-4de0-9eaa-05f11eb78b44","JobId":"long-running-cfn5q","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.435Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e61373b1-9c9a-4de0-9eaa-05f11eb78b44"}
{"level":"error","ts":"2026-07-10T10:43:12.435Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"e61373b1-9c9a-4de0-9eaa-05f11eb78b44","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.435Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2aa6cdfa-8ab8-4afb-8938-0deb01f1c643","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.435Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2aa6cdfa-8ab8-4afb-8938-0deb01f1c643","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.435Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2aa6cdfa-8ab8-4afb-8938-0deb01f1c643","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.435Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2aa6cdfa-8ab8-4afb-8938-0deb01f1c643"}
{"level":"error","ts":"2026-07-10T10:43:12.440Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2aa6cdfa-8ab8-4afb-8938-0deb01f1c643","JobId":"long-running-cfn5q","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.440Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2aa6cdfa-8ab8-4afb-8938-0deb01f1c643"}
{"level":"error","ts":"2026-07-10T10:43:12.440Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2aa6cdfa-8ab8-4afb-8938-0deb01f1c643","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.440Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0756f7af-c1f6-41b4-bb4c-d7d5dbc5fdda","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.440Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0756f7af-c1f6-41b4-bb4c-d7d5dbc5fdda","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.440Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0756f7af-c1f6-41b4-bb4c-d7d5dbc5fdda","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.440Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0756f7af-c1f6-41b4-bb4c-d7d5dbc5fdda"}
{"level":"error","ts":"2026-07-10T10:43:12.445Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0756f7af-c1f6-41b4-bb4c-d7d5dbc5fdda","JobId":"long-running-cfn5q","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.445Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0756f7af-c1f6-41b4-bb4c-d7d5dbc5fdda"}
{"level":"error","ts":"2026-07-10T10:43:12.445Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"0756f7af-c1f6-41b4-bb4c-d7d5dbc5fdda","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.445Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a5101d79-8c7e-477b-8e0c-e1d9cf709611","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.445Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a5101d79-8c7e-477b-8e0c-e1d9cf709611","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.445Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a5101d79-8c7e-477b-8e0c-e1d9cf709611","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.445Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a5101d79-8c7e-477b-8e0c-e1d9cf709611"}
{"level":"error","ts":"2026-07-10T10:43:12.529Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a5101d79-8c7e-477b-8e0c-e1d9cf709611","JobId":"long-running-cfn5q","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.529Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a5101d79-8c7e-477b-8e0c-e1d9cf709611"}
{"level":"error","ts":"2026-07-10T10:43:12.529Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"a5101d79-8c7e-477b-8e0c-e1d9cf709611","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.529Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"83b005fe-1e27-4f14-ba8e-b43c592abb6a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.529Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"83b005fe-1e27-4f14-ba8e-b43c592abb6a","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.529Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"83b005fe-1e27-4f14-ba8e-b43c592abb6a","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.529Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"83b005fe-1e27-4f14-ba8e-b43c592abb6a"}
{"level":"error","ts":"2026-07-10T10:43:12.534Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"83b005fe-1e27-4f14-ba8e-b43c592abb6a","JobId":"long-running-cfn5q","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.534Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"83b005fe-1e27-4f14-ba8e-b43c592abb6a"}
{"level":"error","ts":"2026-07-10T10:43:12.534Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"83b005fe-1e27-4f14-ba8e-b43c592abb6a","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.610Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ce64a99-5dcf-454d-bca1-2189e50399b9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.610Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ce64a99-5dcf-454d-bca1-2189e50399b9","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.610Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ce64a99-5dcf-454d-bca1-2189e50399b9","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.610Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ce64a99-5dcf-454d-bca1-2189e50399b9"}
{"level":"error","ts":"2026-07-10T10:43:12.616Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ce64a99-5dcf-454d-bca1-2189e50399b9","JobId":"long-running-cfn5q","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.616Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ce64a99-5dcf-454d-bca1-2189e50399b9"}
{"level":"error","ts":"2026-07-10T10:43:12.616Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"6ce64a99-5dcf-454d-bca1-2189e50399b9","error":"Job long-running-cfn5q does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.681Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"90a778f0-f8d6-4cd7-adcf-025b4874e65f","DeletionTimestamp":"2026-07-10 10:43:12 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:43:12.681Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"90a778f0-f8d6-4cd7-adcf-025b4874e65f","rayJob":"long-running-cfn5q"}
{"level":"error","ts":"2026-07-10T10:43:12.687Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"90a778f0-f8d6-4cd7-adcf-025b4874e65f","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.687Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"90a778f0-f8d6-4cd7-adcf-025b4874e65f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:43:12.702Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6699b06b-369f-4294-a14a-aa5a720d04ed","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.702Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6699b06b-369f-4294-a14a-aa5a720d04ed","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:43:12.718Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6699b06b-369f-4294-a14a-aa5a720d04ed"}
{"level":"info","ts":"2026-07-10T10:43:12.718Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6699b06b-369f-4294-a14a-aa5a720d04ed","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:43:12.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6699b06b-369f-4294-a14a-aa5a720d04ed","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-bprzq","rayClusterName":"counter-pb7bn","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:12.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6699b06b-369f-4294-a14a-aa5a720d04ed","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9c68e7be-5636-4d93-bffd-eadd935e8ca2","rayCluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"d8af41b8-8e63-4819-b629-2653e8c6b2d6"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"be44b89a-db85-4e3b-8d52-2a888dc13061"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9c68e7be-5636-4d93-bffd-eadd935e8ca2","pod":"long-running-bfn9m-head-8zfts","ip":"10.132.0.28"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9c68e7be-5636-4d93-bffd-eadd935e8ca2","pod":"long-running-bfn9m-small-group-worker-kzbpb","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","namespace":"test-ns-pqpm4","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:43:12.733Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"9c68e7be-5636-4d93-bffd-eadd935e8ca2","rayCluster":"long-running-bfn9m","podIPs":["10.132.0.28","10.133.0.36"]}
{"level":"info","ts":"2026-07-10T10:43:12.736Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"2a1e8e87-70ab-4cc8-8bb7-0a14cc310082"}
{"level":"info","ts":"2026-07-10T10:43:12.736Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.736Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072"}
{"level":"info","ts":"2026-07-10T10:43:12.736Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:43:12.736Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-f7zxt","rayClusterName":"counter-x7tjf","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:12.736Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:43:12.745Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","name":"kuberay-gateway-access","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:12.745Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:43:12.745Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072"}
{"level":"error","ts":"2026-07-10T10:43:12.745Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6842975b-0358-4271-8d21-8f69ab28c072","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.745Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bbf58f97-9b9e-453a-92a2-f263a4441bf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.745Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bbf58f97-9b9e-453a-92a2-f263a4441bf8","oldStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:43:12.745Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bbf58f97-9b9e-453a-92a2-f263a4441bf8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-bprzq","rayClusterName":"counter-pb7bn","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-bprzq","rayClusterName":"counter-pb7bn","jobDeploymentStatus":"Suspending","startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:12.745Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bbf58f97-9b9e-453a-92a2-f263a4441bf8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-07-10T10:43:12.753Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","name":"test-ns-pqpm4-long-running-bfn9m","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:43:12.759Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5f7a7ec-0d0f-499f-aa31-aa73d8eb501c","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.759Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5f7a7ec-0d0f-499f-aa31-aa73d8eb501c","RayCluster":{"name":"counter-pb7bn","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.759Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5f7a7ec-0d0f-499f-aa31-aa73d8eb501c","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:43:12.759Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5f7a7ec-0d0f-499f-aa31-aa73d8eb501c","Kubernetes Job":""}
{"level":"info","ts":"2026-07-10T10:43:12.759Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5f7a7ec-0d0f-499f-aa31-aa73d8eb501c","isJobDeleted":true}
{"level":"info","ts":"2026-07-10T10:43:12.759Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5f7a7ec-0d0f-499f-aa31-aa73d8eb501c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-bprzq","rayClusterName":"counter-pb7bn","jobDeploymentStatus":"Suspending","startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:12.759Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5f7a7ec-0d0f-499f-aa31-aa73d8eb501c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-07-10T10:43:12.760Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","configMap":"kube-rbac-proxy-config-long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.767Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","serviceAccount":"long-running-bfn9m-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:43:12.775Z","logger":"raycluster-resource","msg":"default","name":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.775Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:12.825Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"b0ef9137-0bec-42aa-a6a2-7f66fd488701"}
{"level":"info","ts":"2026-07-10T10:43:12.825Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7d2f13a9-701e-41b5-9bdf-63ef627ea31d"}
{"level":"info","ts":"2026-07-10T10:43:12.825Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"73df8204-cbb2-4759-a60d-8c1a84a0d5ea"}
{"level":"info","ts":"2026-07-10T10:43:12.825Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"3b563d0e-a4a5-42cb-abcd-12130e9cacb7"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"reconcileID":"0bbdd40a-9833-40e0-990d-ddf7d22d560e"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf677aaa-51e3-49c3-a2a1-8b88764b8ebb","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"c20c6047-4469-4889-854c-137a91898a4c","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7","cluster":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7","namespace":"test-ns-pqpm4","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"dce4ab90-46b0-4f89-8167-bf65967c1303"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"3f2c0b89-c3e5-4c46-8cca-a87e864ecd88"}
{"level":"info","ts":"2026-07-10T10:43:12.826Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7a5ea54d-da94-4974-b7f7-5a34c3db0e83","namespace":"test-ns-pqpm4","clusterName":"long-running-bfn9m"}
{"level":"error","ts":"2026-07-10T10:43:12.831Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7","error":"rayclusters.ray.io \"long-running-bfn9m\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.831Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7"}
{"level":"error","ts":"2026-07-10T10:43:12.831Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"882485bc-4047-4e9f-a353-c8740b1b85c7","error":"rayclusters.ray.io \"long-running-bfn9m\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:12.832Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"253449c9-9aad-4129-b14f-911302464771","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.832Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"253449c9-9aad-4129-b14f-911302464771"}
{"level":"info","ts":"2026-07-10T10:43:12.832Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"253449c9-9aad-4129-b14f-911302464771","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:43:12.832Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"253449c9-9aad-4129-b14f-911302464771","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:12.837Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"0838b98f-1586-4d87-95e3-40a94886f20b","namespacedName":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:12.837Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"0838b98f-1586-4d87-95e3-40a94886f20b"}
{"level":"info","ts":"2026-07-10T10:43:12.837Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"0838b98f-1586-4d87-95e3-40a94886f20b","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:43:12.837Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"0838b98f-1586-4d87-95e3-40a94886f20b","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:12.860Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"7a5ea54d-da94-4974-b7f7-5a34c3db0e83","namespace":"test-ns-pqpm4","clusterName":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:43:12.937Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-pqpm4"},"reconcileID":"040cedf6-0c6a-4c8e-b70d-acb07eabe61e"}
{"level":"info","ts":"2026-07-10T10:43:13.723Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"452050b2-eb14-47b0-ac9e-bb6b911214d7","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:13.723Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"452050b2-eb14-47b0-ac9e-bb6b911214d7"}
{"level":"info","ts":"2026-07-10T10:43:13.723Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"452050b2-eb14-47b0-ac9e-bb6b911214d7","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:13.723Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"452050b2-eb14-47b0-ac9e-bb6b911214d7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-07-10T10:43:13.735Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e8935cd-3367-429c-92b3-08cb29458de5","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:13.735Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e8935cd-3367-429c-92b3-08cb29458de5"}
{"level":"info","ts":"2026-07-10T10:43:13.735Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e8935cd-3367-429c-92b3-08cb29458de5","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:43:13.735Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e8935cd-3367-429c-92b3-08cb29458de5","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-10T10:43:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:43:13.735Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e8935cd-3367-429c-92b3-08cb29458de5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:43:13.747Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ac1d39c-7674-4462-a67b-15f1e143d912","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:13.747Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ac1d39c-7674-4462-a67b-15f1e143d912","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.747Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ac1d39c-7674-4462-a67b-15f1e143d912","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.747Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ac1d39c-7674-4462-a67b-15f1e143d912","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.752Z","logger":"raycluster-resource","msg":"default","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.752Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-5wbjw","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ac1d39c-7674-4462-a67b-15f1e143d912","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"75229ed2-4951-411c-a3a0-ae21cdff0007","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","mode":"IntegratedOAuth","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ac1d39c-7674-4462-a67b-15f1e143d912"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ac1d39c-7674-4462-a67b-15f1e143d912","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"be3fdccb-66cb-4ad8-ba1f-2a27e4ea6ae4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"be3fdccb-66cb-4ad8-ba1f-2a27e4ea6ae4","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"be3fdccb-66cb-4ad8-ba1f-2a27e4ea6ae4","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"be3fdccb-66cb-4ad8-ba1f-2a27e4ea6ae4"}
{"level":"info","ts":"2026-07-10T10:43:13.767Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"be3fdccb-66cb-4ad8-ba1f-2a27e4ea6ae4","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:13.774Z","logger":"raycluster-resource","msg":"default","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.774Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-5wbjw","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:43:13.775Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","name":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:13.775Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:13.775Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.775Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:13.776Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.776Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","serviceSelectors":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/identifier":"counter-5wbjw-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.776Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.776Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","error":"unable to find head service. cluster name counter-5wbjw, filter labels [test-ns-pqpm4 map[ray.io/cluster:counter-5wbjw ray.io/identifier:counter-5wbjw-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20"}
{"level":"error","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"3459d1d0-31c5-4ce1-9da0-01b6e911eb20","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","name":"counter-5wbjw","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"75229ed2-4951-411c-a3a0-ae21cdff0007","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.825Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"10dd6c9b-9e9d-435e-8cce-ebfe009865dd","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.826Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dbbcaefa-9cc4-4ccc-98bc-3caf68ca273e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:13.826Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dbbcaefa-9cc4-4ccc-98bc-3caf68ca273e","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.826Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dbbcaefa-9cc4-4ccc-98bc-3caf68ca273e","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.826Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dbbcaefa-9cc4-4ccc-98bc-3caf68ca273e"}
{"level":"info","ts":"2026-07-10T10:43:13.826Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dbbcaefa-9cc4-4ccc-98bc-3caf68ca273e","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:13.833Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","name":"counter-5wbjw-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","name":"counter-5wbjw","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5wbjw\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"counter-5wbjw"}}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299"}
{"level":"error","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"cd14c0cf-142f-4061-af1b-2fc3e3b03299","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f"}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f"}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:43:13.834Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","name":"counter-5wbjw","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:13.836Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:43:13.836Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.841Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"10dd6c9b-9e9d-435e-8cce-ebfe009865dd","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:13.842Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","name":"kuberay-gateway-access","namespace":"test-ns-pqpm4","operation":"created"}
{"level":"info","ts":"2026-07-10T10:43:13.846Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"da526210-b83a-4359-8e21-1f01ea0cda7c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:13.846Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"da526210-b83a-4359-8e21-1f01ea0cda7c","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.846Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"da526210-b83a-4359-8e21-1f01ea0cda7c","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.846Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"da526210-b83a-4359-8e21-1f01ea0cda7c"}
{"level":"info","ts":"2026-07-10T10:43:13.846Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"da526210-b83a-4359-8e21-1f01ea0cda7c","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:13.846Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f"}
{"level":"error","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"425fc1fa-8ab8-4ee5-ab35-f1775ed5f86f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15"}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15"}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15"}
{"level":"error","ts":"2026-07-10T10:43:13.847Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f228a3d9-d232-40cb-81ff-d19aed558b15","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.848Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710"}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710"}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710"}
{"level":"info","ts":"2026-07-10T10:43:13.925Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","name":"test-ns-pqpm4-counter-5wbjw","operation":"created"}
{"level":"error","ts":"2026-07-10T10:43:13.925Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ce280384-6de4-4d39-83df-5d56f74a5710","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.930Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"10dd6c9b-9e9d-435e-8cce-ebfe009865dd","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:13.930Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"10dd6c9b-9e9d-435e-8cce-ebfe009865dd","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.930Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"bdf76997-c851-4dce-8bcc-3b9294713050","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.931Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","name":"kube-rbac-proxy-config-counter-5wbjw","operation":"created"}
{"level":"info","ts":"2026-07-10T10:43:13.934Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","rayCluster":"counter-5wbjw","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:43:13.934Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-5wbjw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.934Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a"}
{"level":"error","ts":"2026-07-10T10:43:13.934Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"61ab42d6-e2b0-4ea2-9a14-4ddb92fc109a","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-5wbjw\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.934Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"14abc1a2-6a15-42f1-a3f3-2976bfd3e63d","rayCluster":"counter-5wbjw"}
{"level":"error","ts":"2026-07-10T10:43:13.938Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5wbjw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.938Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba"}
{"level":"error","ts":"2026-07-10T10:43:13.938Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"09d26f78-b749-4b12-b625-909eee11dbba","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5wbjw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:13.938Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e1baf046-7c46-4fcc-b519-1b35dccc5902","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:13.938Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e1baf046-7c46-4fcc-b519-1b35dccc5902","mode":"IntegratedOAuth","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.938Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e1baf046-7c46-4fcc-b519-1b35dccc5902","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:13.947Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"bdf76997-c851-4dce-8bcc-3b9294713050","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:14.025Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"14abc1a2-6a15-42f1-a3f3-2976bfd3e63d","error":"certificates.cert-manager.io \"ray-worker-cert-counter-5wbjw\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-07-10T10:43:14.025Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"14abc1a2-6a15-42f1-a3f3-2976bfd3e63d","error":"certificates.cert-manager.io \"ray-worker-cert-counter-5wbjw\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"14abc1a2-6a15-42f1-a3f3-2976bfd3e63d"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249"}
{"level":"error","ts":"2026-07-10T10:43:14.025Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"14abc1a2-6a15-42f1-a3f3-2976bfd3e63d","error":"certificates.cert-manager.io \"ray-worker-cert-counter-5wbjw\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:43:14.025Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"298b4d82-93ee-4760-aee1-1de517b5a249","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1bfaa4b3-5308-41b2-a925-a21e9b5421cd","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1bfaa4b3-5308-41b2-a925-a21e9b5421cd"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e1baf046-7c46-4fcc-b519-1b35dccc5902","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.025Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b7fbb702-9465-4204-ae45-c4a5e7342fbf","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b7fbb702-9465-4204-ae45-c4a5e7342fbf","mode":"IntegratedOAuth","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b7fbb702-9465-4204-ae45-c4a5e7342fbf","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"0d64cfd5-fae3-4be1-ab45-44eb16abf5d0","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"0d64cfd5-fae3-4be1-ab45-44eb16abf5d0"}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0a990dfa-d93d-4a9a-8378-67e21ddc3fd1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0a990dfa-d93d-4a9a-8378-67e21ddc3fd1","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0a990dfa-d93d-4a9a-8378-67e21ddc3fd1","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0a990dfa-d93d-4a9a-8378-67e21ddc3fd1"}
{"level":"info","ts":"2026-07-10T10:43:14.026Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0a990dfa-d93d-4a9a-8378-67e21ddc3fd1","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:14.032Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"bdf76997-c851-4dce-8bcc-3b9294713050","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:14.032Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"bdf76997-c851-4dce-8bcc-3b9294713050","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.032Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ac7f6143-64ef-47d4-be7c-b3eb101f38e1","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.035Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"37412885-cb0f-4dd9-996d-b0c2646d97d6","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.035Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"37412885-cb0f-4dd9-996d-b0c2646d97d6"}
{"level":"info","ts":"2026-07-10T10:43:14.037Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b7fbb702-9465-4204-ae45-c4a5e7342fbf","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.046Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ac7f6143-64ef-47d4-be7c-b3eb101f38e1","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:14.125Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ac7f6143-64ef-47d4-be7c-b3eb101f38e1","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:14.125Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ac7f6143-64ef-47d4-be7c-b3eb101f38e1","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee"}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee"}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee"}
{"level":"error","ts":"2026-07-10T10:43:14.186Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"14f75312-f127-4e49-a199-6c7bd9e5ceee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7"}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7"}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7"}
{"level":"error","ts":"2026-07-10T10:43:14.507Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"788fcb5c-6c87-4090-bea0-eee5160892b7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:15.147Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541"}
{"level":"info","ts":"2026-07-10T10:43:15.147Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541"}
{"level":"info","ts":"2026-07-10T10:43:15.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:15.147Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","error":"head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"error","ts":"2026-07-10T10:43:15.147Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:15.147Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found"}
{"level":"info","ts":"2026-07-10T10:43:15.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:15.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:15.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","filter labels":["test-ns-pqpm4",{"ray.io/cluster":"counter-5wbjw","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:43:15.148Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541"}
{"level":"error","ts":"2026-07-10T10:43:15.148Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"29dcf44a-a0a7-4978-af78-f04b58a2f541","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5wbjw not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:15.760Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"504d76d1-a9e0-4bce-a42c-1d31cbd930b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:15.760Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"504d76d1-a9e0-4bce-a42c-1d31cbd930b1","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:15.760Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"504d76d1-a9e0-4bce-a42c-1d31cbd930b1","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:15.760Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"504d76d1-a9e0-4bce-a42c-1d31cbd930b1"}
{"level":"info","ts":"2026-07-10T10:43:15.760Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"504d76d1-a9e0-4bce-a42c-1d31cbd930b1","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","mode":"IntegratedOAuth","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","cluster":"counter-5wbjw","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","cluster":"counter-5wbjw","authType":"IntegratedOAuth","serviceAccount":"counter-5wbjw-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-5wbjw","ray.io/group":"headgroup","ray.io/identifier":"counter-5wbjw-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:43:16.428Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","enabled":true}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","name":"counter-5wbjw-head-n5slx"}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8"}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","nodeType":"worker","rayStartParams":{"address":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","rayStartCmd":"ray start  --address=counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:43:16.445Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","enabled":true}
{"level":"info","ts":"2026-07-10T10:43:16.458Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","name":"counter-5wbjw-small-group-worker-m8qnb"}
{"level":"info","ts":"2026-07-10T10:43:16.459Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.459Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.459Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-5wbjw-head-svc"},"newHead":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:43:16.459Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"adb5cf84-9039-4b94-b473-236a7119a8f8","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.472Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-5wbjw-head-svc"},"newHead":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca1d19c3-695e-46e1-a01d-f318b2ad7aef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca1d19c3-695e-46e1-a01d-f318b2ad7aef","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca1d19c3-695e-46e1-a01d-f318b2ad7aef","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1340e089-37b3-4ff4-aa43-32a691e25134","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"60d202ed-fcac-432c-bd74-7065089ce499","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca1d19c3-695e-46e1-a01d-f318b2ad7aef"}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca1d19c3-695e-46e1-a01d-f318b2ad7aef","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:16.473Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"60d202ed-fcac-432c-bd74-7065089ce499","rayCluster":"counter-5wbjw","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:43:16.525Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","name":"counter-5wbjw","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5wbjw\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"counter-5wbjw"}}
{"level":"info","ts":"2026-07-10T10:43:16.525Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2"}
{"level":"error","ts":"2026-07-10T10:43:16.525Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"679c6b89-9ad0-41d3-903d-b3cfcaf371d2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5wbjw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.526Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:43:16.527Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"92ef5121-5b30-47bb-b8c9-3430ec9d24c4","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:16.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1340e089-37b3-4ff4-aa43-32a691e25134","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:16.637Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280"}
{"level":"info","ts":"2026-07-10T10:43:16.637Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280"}
{"level":"info","ts":"2026-07-10T10:43:16.637Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:16.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:16.640Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1340e089-37b3-4ff4-aa43-32a691e25134","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1340e089-37b3-4ff4-aa43-32a691e25134","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"a25f2a2e-da54-4a38-a8d1-2e306715055e","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ebf508f0-4086-429f-9cac-038b789e23bd","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5681a3d-b8b4-430c-86a1-1fb34dd7811b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5681a3d-b8b4-430c-86a1-1fb34dd7811b","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5681a3d-b8b4-430c-86a1-1fb34dd7811b","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ebf508f0-4086-429f-9cac-038b789e23bd","rayCluster":"counter-5wbjw","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5681a3d-b8b4-430c-86a1-1fb34dd7811b"}
{"level":"info","ts":"2026-07-10T10:43:16.725Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a5681a3d-b8b4-430c-86a1-1fb34dd7811b","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:16.735Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","name":"counter-5wbjw","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5wbjw\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pqpm4","name":"counter-5wbjw"}}
{"level":"info","ts":"2026-07-10T10:43:16.735Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280"}
{"level":"error","ts":"2026-07-10T10:43:16.735Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"bc941818-1839-4e41-b422-20f6b3596280","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5wbjw\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:43:16.739Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"a25f2a2e-da54-4a38-a8d1-2e306715055e","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2"}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2"}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.740Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.741Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.741Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:16.741Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"2313ab14-82cd-4a7d-be40-0961b27e0ed2","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:16.751Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"a25f2a2e-da54-4a38-a8d1-2e306715055e","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:16.751Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"a25f2a2e-da54-4a38-a8d1-2e306715055e","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:16.907Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f672cf25-85e2-43b0-8c50-7255cf7a4766","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:16.925Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"b73ebf88-5692-419d-8159-723075dbb83c","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:17.428Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"6d3e41c8-fc3e-48fd-b676-b59399c0f4df","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:17.852Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a"}
{"level":"info","ts":"2026-07-10T10:43:17.852Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a"}
{"level":"info","ts":"2026-07-10T10:43:17.852Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:17.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"newHead":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"}}
{"level":"info","ts":"2026-07-10T10:43:17.853Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"eab16be3-7cc0-4238-962f-46bcda51b12a","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7c2aab2a-355e-4092-a6ae-df8fc31aaa75","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"27c541c4-6b45-47ac-8a7c-78521904c0f1","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bb957e24-e8d8-495e-8b5e-8caa256f4613","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bb957e24-e8d8-495e-8b5e-8caa256f4613","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bb957e24-e8d8-495e-8b5e-8caa256f4613","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"27c541c4-6b45-47ac-8a7c-78521904c0f1","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"27c541c4-6b45-47ac-8a7c-78521904c0f1","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bb957e24-e8d8-495e-8b5e-8caa256f4613"}
{"level":"info","ts":"2026-07-10T10:43:17.869Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bb957e24-e8d8-495e-8b5e-8caa256f4613","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:17.880Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"27c541c4-6b45-47ac-8a7c-78521904c0f1","certificate":"ray-head-cert-counter-5wbjw","dnsNames":["counter-5wbjw-head-svc","localhost","counter-5wbjw-head-svc.test-ns-pqpm4.svc","counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local"],"ipAddresses":["10.132.0.29","10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:43:17.887Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7c2aab2a-355e-4092-a6ae-df8fc31aaa75","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:17.906Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"27c541c4-6b45-47ac-8a7c-78521904c0f1","certificate":"ray-worker-cert-counter-5wbjw","dnsNames":["counter-5wbjw-worker-svc","localhost","counter-5wbjw-small-group","counter-5wbjw-worker-svc.test-ns-pqpm4.svc","counter-5wbjw-worker-svc.test-ns-pqpm4.svc.cluster.local","*.counter-5wbjw-worker-svc.test-ns-pqpm4.svc","*.counter-5wbjw-worker-svc.test-ns-pqpm4.svc.cluster.local","*-worker-*.test-ns-pqpm4.svc","*-worker-*.test-ns-pqpm4.svc.cluster.local"],"ipAddresses":["10.132.0.29","10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:43:17.906Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"27c541c4-6b45-47ac-8a7c-78521904c0f1"}
{"level":"info","ts":"2026-07-10T10:43:17.912Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7c2aab2a-355e-4092-a6ae-df8fc31aaa75","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:17.912Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7c2aab2a-355e-4092-a6ae-df8fc31aaa75","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:18.761Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0f1fd195-04e6-4081-8eed-5653292a8eaa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:18.761Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0f1fd195-04e6-4081-8eed-5653292a8eaa","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:18.761Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0f1fd195-04e6-4081-8eed-5653292a8eaa","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:18.761Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0f1fd195-04e6-4081-8eed-5653292a8eaa"}
{"level":"info","ts":"2026-07-10T10:43:18.761Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0f1fd195-04e6-4081-8eed-5653292a8eaa","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:19.869Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a363faee-515d-457d-b284-5637d72863a0","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:21.762Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"26826110-8e24-4cc6-89ac-ca05c9ad5072","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:21.762Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"26826110-8e24-4cc6-89ac-ca05c9ad5072","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:21.762Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"26826110-8e24-4cc6-89ac-ca05c9ad5072","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:21.762Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"26826110-8e24-4cc6-89ac-ca05c9ad5072"}
{"level":"info","ts":"2026-07-10T10:43:21.762Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"26826110-8e24-4cc6-89ac-ca05c9ad5072","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","podName":"counter-5wbjw-head-n5slx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:22.874Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0408d626-b1af-4753-b7e1-b85b27f4b84b","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:23.886Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d"}
{"level":"info","ts":"2026-07-10T10:43:23.886Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d"}
{"level":"info","ts":"2026-07-10T10:43:23.886Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","podName":"counter-5wbjw-head-n5slx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","podName":"counter-5wbjw-head-n5slx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:43:23.887Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06018e4c-1546-424c-821d-bf4f74dfc31d","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"37780549-5e92-4c38-bea9-a11c799f2017","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f043b336-5316-4701-8fd2-1cf7e98cf5ec","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4eb97f65-50d3-4b30-9b6f-6920fb76cb65","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4eb97f65-50d3-4b30-9b6f-6920fb76cb65","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4eb97f65-50d3-4b30-9b6f-6920fb76cb65","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f043b336-5316-4701-8fd2-1cf7e98cf5ec","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f043b336-5316-4701-8fd2-1cf7e98cf5ec","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f043b336-5316-4701-8fd2-1cf7e98cf5ec","rayCluster":"counter-5wbjw","podIPs":["10.133.0.38","10.132.0.29"]}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4eb97f65-50d3-4b30-9b6f-6920fb76cb65"}
{"level":"info","ts":"2026-07-10T10:43:23.898Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4eb97f65-50d3-4b30-9b6f-6920fb76cb65","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:23.908Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"37780549-5e92-4c38-bea9-a11c799f2017","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:23.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"37780549-5e92-4c38-bea9-a11c799f2017","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:23.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"37780549-5e92-4c38-bea9-a11c799f2017","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:24.763Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"9d900df6-f7e4-4acd-a2f6-1d1b50d956ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:24.763Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"9d900df6-f7e4-4acd-a2f6-1d1b50d956ee","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:24.763Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"9d900df6-f7e4-4acd-a2f6-1d1b50d956ee","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:24.763Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"9d900df6-f7e4-4acd-a2f6-1d1b50d956ee"}
{"level":"info","ts":"2026-07-10T10:43:24.763Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"9d900df6-f7e4-4acd-a2f6-1d1b50d956ee","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245"}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245"}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","podName":"counter-5wbjw-head-n5slx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","podName":"counter-5wbjw-head-n5slx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:25.899Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"7c1c73de-b399-40a1-afd7-a3fed60f9245","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:27.764Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3719a82d-9754-48a8-b88b-1b80d83c6204","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:27.764Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3719a82d-9754-48a8-b88b-1b80d83c6204","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:27.764Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3719a82d-9754-48a8-b88b-1b80d83c6204","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:27.764Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3719a82d-9754-48a8-b88b-1b80d83c6204"}
{"level":"info","ts":"2026-07-10T10:43:27.764Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3719a82d-9754-48a8-b88b-1b80d83c6204","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:30.764Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"34f21682-50c9-4a3c-9932-0ca91d241192","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:30.764Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"34f21682-50c9-4a3c-9932-0ca91d241192","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:30.764Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"34f21682-50c9-4a3c-9932-0ca91d241192","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:30.764Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"34f21682-50c9-4a3c-9932-0ca91d241192"}
{"level":"info","ts":"2026-07-10T10:43:30.764Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"34f21682-50c9-4a3c-9932-0ca91d241192","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:33.765Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a1c31cee-f8a2-4d06-a085-b677a59b8f09","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:33.765Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a1c31cee-f8a2-4d06-a085-b677a59b8f09","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:33.765Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a1c31cee-f8a2-4d06-a085-b677a59b8f09","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:33.765Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a1c31cee-f8a2-4d06-a085-b677a59b8f09"}
{"level":"info","ts":"2026-07-10T10:43:33.765Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"a1c31cee-f8a2-4d06-a085-b677a59b8f09","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:43:34.918Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"ed65692e-939d-4ea7-9659-794c9b40edf2","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:43:34.930Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1391b50d-fad9-4452-a192-96e42a21563e","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:34.930Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7e453c3a-c0a4-44ea-bd62-542e2e517140","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1391b50d-fad9-4452-a192-96e42a21563e","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1391b50d-fad9-4452-a192-96e42a21563e","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1391b50d-fad9-4452-a192-96e42a21563e","rayCluster":"counter-5wbjw","podIPs":["10.132.0.29","10.133.0.38"]}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"de0d673a-a5fd-403b-bfd6-7128d2d9ccc0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"de0d673a-a5fd-403b-bfd6-7128d2d9ccc0","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"de0d673a-a5fd-403b-bfd6-7128d2d9ccc0","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"de0d673a-a5fd-403b-bfd6-7128d2d9ccc0"}
{"level":"info","ts":"2026-07-10T10:43:34.931Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"de0d673a-a5fd-403b-bfd6-7128d2d9ccc0","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:34.950Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7e453c3a-c0a4-44ea-bd62-542e2e517140","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:43:34.960Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7e453c3a-c0a4-44ea-bd62-542e2e517140","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:43:34.960Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"7e453c3a-c0a4-44ea-bd62-542e2e517140","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:36.765Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"187fdf4b-6afb-4fd0-9d07-a56a39af7e5f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:36.766Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"187fdf4b-6afb-4fd0-9d07-a56a39af7e5f","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:36.766Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"187fdf4b-6afb-4fd0-9d07-a56a39af7e5f","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:36.766Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"187fdf4b-6afb-4fd0-9d07-a56a39af7e5f"}
{"level":"info","ts":"2026-07-10T10:43:36.766Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"187fdf4b-6afb-4fd0-9d07-a56a39af7e5f","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:43:36.931Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"0aa67b8b-6dcb-4172-9cc2-e81c707edd76","seconds":300}
{"level":"info","ts":"2026-07-10T10:43:39.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e63053a2-8378-40bd-91f5-51b50143306d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:39.767Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e63053a2-8378-40bd-91f5-51b50143306d","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:39.767Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e63053a2-8378-40bd-91f5-51b50143306d","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:39.767Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e63053a2-8378-40bd-91f5-51b50143306d"}
{"level":"info","ts":"2026-07-10T10:43:39.767Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e63053a2-8378-40bd-91f5-51b50143306d","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:42.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"599c380a-6a64-4731-8567-5b734fd7d08c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:42.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"599c380a-6a64-4731-8567-5b734fd7d08c","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:42.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"599c380a-6a64-4731-8567-5b734fd7d08c","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:42.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"599c380a-6a64-4731-8567-5b734fd7d08c"}
{"level":"info","ts":"2026-07-10T10:43:42.768Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"599c380a-6a64-4731-8567-5b734fd7d08c","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:44.035Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"fff1aaac-b35e-46cf-8a10-bb8cfa36754f","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:44.036Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"fff1aaac-b35e-46cf-8a10-bb8cfa36754f","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:43:44.036Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"fff1aaac-b35e-46cf-8a10-bb8cfa36754f","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:43:44.036Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"fff1aaac-b35e-46cf-8a10-bb8cfa36754f","rayCluster":"counter-5wbjw","podIPs":["10.132.0.29","10.133.0.38"]}
{"level":"info","ts":"2026-07-10T10:43:45.769Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"404e04fb-e107-42d3-9472-3c608a4ef577","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:45.769Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"404e04fb-e107-42d3-9472-3c608a4ef577","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:45.769Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"404e04fb-e107-42d3-9472-3c608a4ef577","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:45.769Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"404e04fb-e107-42d3-9472-3c608a4ef577"}
{"level":"info","ts":"2026-07-10T10:43:45.769Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"404e04fb-e107-42d3-9472-3c608a4ef577","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:48.770Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e5d45c13-3021-474b-9335-0590123172e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:48.770Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e5d45c13-3021-474b-9335-0590123172e3","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:48.770Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e5d45c13-3021-474b-9335-0590123172e3","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:48.770Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e5d45c13-3021-474b-9335-0590123172e3"}
{"level":"info","ts":"2026-07-10T10:43:48.770Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e5d45c13-3021-474b-9335-0590123172e3","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:51.771Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"72393517-8122-4359-9fc2-a8e574ab25f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:51.771Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"72393517-8122-4359-9fc2-a8e574ab25f1","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:51.771Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"72393517-8122-4359-9fc2-a8e574ab25f1","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:51.771Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"72393517-8122-4359-9fc2-a8e574ab25f1"}
{"level":"info","ts":"2026-07-10T10:43:51.771Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"72393517-8122-4359-9fc2-a8e574ab25f1","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:54.772Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6594fe77-c872-4a7a-9047-aa7f37f96efe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:54.772Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6594fe77-c872-4a7a-9047-aa7f37f96efe","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:54.772Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6594fe77-c872-4a7a-9047-aa7f37f96efe","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:54.772Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6594fe77-c872-4a7a-9047-aa7f37f96efe"}
{"level":"info","ts":"2026-07-10T10:43:54.772Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6594fe77-c872-4a7a-9047-aa7f37f96efe","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:43:57.773Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d3e747c9-b22d-41b7-ae3a-03e2df5bc8ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:43:57.773Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d3e747c9-b22d-41b7-ae3a-03e2df5bc8ec","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:43:57.773Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d3e747c9-b22d-41b7-ae3a-03e2df5bc8ec","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:43:57.773Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d3e747c9-b22d-41b7-ae3a-03e2df5bc8ec"}
{"level":"info","ts":"2026-07-10T10:43:57.773Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d3e747c9-b22d-41b7-ae3a-03e2df5bc8ec","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:00.774Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ef22847-f999-4f9c-9fb8-9496f7523c50","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:00.774Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ef22847-f999-4f9c-9fb8-9496f7523c50","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:00.774Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ef22847-f999-4f9c-9fb8-9496f7523c50","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:00.774Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ef22847-f999-4f9c-9fb8-9496f7523c50"}
{"level":"info","ts":"2026-07-10T10:44:00.774Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8ef22847-f999-4f9c-9fb8-9496f7523c50","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:03.775Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"efd58db2-6601-40a8-8466-6254d9668918","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:03.775Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"efd58db2-6601-40a8-8466-6254d9668918","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:03.775Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"efd58db2-6601-40a8-8466-6254d9668918","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:03.775Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"efd58db2-6601-40a8-8466-6254d9668918"}
{"level":"info","ts":"2026-07-10T10:44:03.775Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"efd58db2-6601-40a8-8466-6254d9668918","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:06.776Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b4d74429-a8e4-4fbd-b69b-a2eae21916ef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:06.776Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b4d74429-a8e4-4fbd-b69b-a2eae21916ef","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:06.776Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b4d74429-a8e4-4fbd-b69b-a2eae21916ef","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:06.776Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b4d74429-a8e4-4fbd-b69b-a2eae21916ef"}
{"level":"info","ts":"2026-07-10T10:44:06.776Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b4d74429-a8e4-4fbd-b69b-a2eae21916ef","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:09.777Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca8f7982-b3aa-40a8-a4ce-809ccf4b68ca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:09.777Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca8f7982-b3aa-40a8-a4ce-809ccf4b68ca","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:09.777Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca8f7982-b3aa-40a8-a4ce-809ccf4b68ca","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:09.777Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca8f7982-b3aa-40a8-a4ce-809ccf4b68ca"}
{"level":"info","ts":"2026-07-10T10:44:09.777Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ca8f7982-b3aa-40a8-a4ce-809ccf4b68ca","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:10.362Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"fb45e23b-5d35-4673-b386-302c7ad28515","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:44:10.362Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"fb45e23b-5d35-4673-b386-302c7ad28515","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:44:10.363Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"fb45e23b-5d35-4673-b386-302c7ad28515","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:44:10.363Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"fb45e23b-5d35-4673-b386-302c7ad28515","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:44:12.127Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"d9c200b0-939d-421a-831e-acdb48f4e724","namespace":"test-ns-pqpm4","clusterName":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:44:12.127Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-bfn9m","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"long-running-bfn9m","reconcileID":"d9c200b0-939d-421a-831e-acdb48f4e724","namespace":"test-ns-pqpm4","clusterName":"long-running-bfn9m"}
{"level":"info","ts":"2026-07-10T10:44:12.777Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3592d985-6154-47f1-b6b2-bafa9e0a489d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:12.777Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3592d985-6154-47f1-b6b2-bafa9e0a489d","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:12.777Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3592d985-6154-47f1-b6b2-bafa9e0a489d","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:12.777Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3592d985-6154-47f1-b6b2-bafa9e0a489d"}
{"level":"info","ts":"2026-07-10T10:44:12.777Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3592d985-6154-47f1-b6b2-bafa9e0a489d","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:15.778Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b87b58e8-2573-4a25-9b95-ba5045edbcf7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:15.779Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b87b58e8-2573-4a25-9b95-ba5045edbcf7","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:15.779Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b87b58e8-2573-4a25-9b95-ba5045edbcf7","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:15.779Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b87b58e8-2573-4a25-9b95-ba5045edbcf7"}
{"level":"info","ts":"2026-07-10T10:44:15.779Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b87b58e8-2573-4a25-9b95-ba5045edbcf7","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:18.779Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"fbed814b-23a0-44df-8b29-93db510c418e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:18.779Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"fbed814b-23a0-44df-8b29-93db510c418e","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:18.779Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"fbed814b-23a0-44df-8b29-93db510c418e","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:18.779Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"fbed814b-23a0-44df-8b29-93db510c418e"}
{"level":"info","ts":"2026-07-10T10:44:18.779Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"fbed814b-23a0-44df-8b29-93db510c418e","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:21.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e74fc3d-82f4-409d-9c10-7ae72df8d104","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:21.780Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e74fc3d-82f4-409d-9c10-7ae72df8d104","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:21.780Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e74fc3d-82f4-409d-9c10-7ae72df8d104","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:21.780Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e74fc3d-82f4-409d-9c10-7ae72df8d104"}
{"level":"info","ts":"2026-07-10T10:44:21.780Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e74fc3d-82f4-409d-9c10-7ae72df8d104","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:22.715Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc"}
{"level":"info","ts":"2026-07-10T10:44:22.715Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc"}
{"level":"info","ts":"2026-07-10T10:44:22.715Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:44:22.716Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"a8a41f8f-5e8a-4ba1-972f-46a8354af3fc","seconds":300}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1"}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1"}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:44:23.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:44:23.720Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:44:23.720Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"72f47c4f-667e-4c2c-b4de-68776f3010f1","seconds":300}
{"level":"info","ts":"2026-07-10T10:44:24.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ee475447-d9f8-4c23-a056-04cf51952ef5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:24.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ee475447-d9f8-4c23-a056-04cf51952ef5","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:24.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ee475447-d9f8-4c23-a056-04cf51952ef5","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:24.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ee475447-d9f8-4c23-a056-04cf51952ef5"}
{"level":"info","ts":"2026-07-10T10:44:24.781Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ee475447-d9f8-4c23-a056-04cf51952ef5","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:27.782Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d71bab45-5485-4c3d-8772-b3bb7e17acf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:27.782Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d71bab45-5485-4c3d-8772-b3bb7e17acf8","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:27.782Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d71bab45-5485-4c3d-8772-b3bb7e17acf8","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:27.782Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d71bab45-5485-4c3d-8772-b3bb7e17acf8"}
{"level":"info","ts":"2026-07-10T10:44:27.782Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"d71bab45-5485-4c3d-8772-b3bb7e17acf8","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:30.783Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f24adb98-ec89-4c80-b3b8-fb0db23d69d2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:30.783Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f24adb98-ec89-4c80-b3b8-fb0db23d69d2","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:30.783Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f24adb98-ec89-4c80-b3b8-fb0db23d69d2","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:30.783Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f24adb98-ec89-4c80-b3b8-fb0db23d69d2"}
{"level":"info","ts":"2026-07-10T10:44:30.783Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f24adb98-ec89-4c80-b3b8-fb0db23d69d2","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:32.761Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8"}
{"level":"info","ts":"2026-07-10T10:44:32.761Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8"}
{"level":"info","ts":"2026-07-10T10:44:32.761Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:44:32.762Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"17bdaf14-abef-48ed-88b9-8869f9d7a0a8","seconds":300}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62"}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62"}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:44:33.768Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"f3f08158-f37a-4bfa-9c21-0f35410a7c62","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:43:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ca654c64-f5ab-45e0-bf27-ca7c7871ee52","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"95ddf37e-1228-4e27-8815-0e59252d7890","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"78d99abf-8882-41c2-b603-b6f2ee61cf17","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"78d99abf-8882-41c2-b603-b6f2ee61cf17","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"78d99abf-8882-41c2-b603-b6f2ee61cf17","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ca654c64-f5ab-45e0-bf27-ca7c7871ee52","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ca654c64-f5ab-45e0-bf27-ca7c7871ee52","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"ca654c64-f5ab-45e0-bf27-ca7c7871ee52","rayCluster":"counter-5wbjw","podIPs":["10.132.0.29","10.133.0.38"]}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"78d99abf-8882-41c2-b603-b6f2ee61cf17"}
{"level":"info","ts":"2026-07-10T10:44:33.781Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"78d99abf-8882-41c2-b603-b6f2ee61cf17","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:33.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8f1f3d22-8ef5-4fb4-82b8-0cdf978470a1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:33.784Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8f1f3d22-8ef5-4fb4-82b8-0cdf978470a1","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:33.784Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8f1f3d22-8ef5-4fb4-82b8-0cdf978470a1","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:33.784Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8f1f3d22-8ef5-4fb4-82b8-0cdf978470a1"}
{"level":"info","ts":"2026-07-10T10:44:33.784Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8f1f3d22-8ef5-4fb4-82b8-0cdf978470a1","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:33.797Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"95ddf37e-1228-4e27-8815-0e59252d7890","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:44:33.808Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"95ddf37e-1228-4e27-8815-0e59252d7890","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:44:33.808Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"95ddf37e-1228-4e27-8815-0e59252d7890","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e"}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e"}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:44:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:35.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:44:35.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","podName":"counter-5wbjw-small-group-worker-m8qnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:43:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:44:35.781Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:44:35.781Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"881444a4-0617-4d27-9485-560dbc1a402e","seconds":300}
{"level":"info","ts":"2026-07-10T10:44:36.785Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6a01df-ab03-4896-912b-87a20837ff71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:36.785Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6a01df-ab03-4896-912b-87a20837ff71","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:36.785Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6a01df-ab03-4896-912b-87a20837ff71","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:36.785Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6a01df-ab03-4896-912b-87a20837ff71"}
{"level":"info","ts":"2026-07-10T10:44:36.785Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6a01df-ab03-4896-912b-87a20837ff71","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:39.786Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"db8946ec-286c-41e0-8560-729048f9d6ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:39.786Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"db8946ec-286c-41e0-8560-729048f9d6ad","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:39.786Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"db8946ec-286c-41e0-8560-729048f9d6ad","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:39.786Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"db8946ec-286c-41e0-8560-729048f9d6ad"}
{"level":"info","ts":"2026-07-10T10:44:39.786Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"db8946ec-286c-41e0-8560-729048f9d6ad","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:42.787Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e302a81-968e-4f07-89c1-9a1789df00c3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:42.787Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e302a81-968e-4f07-89c1-9a1789df00c3","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:42.787Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e302a81-968e-4f07-89c1-9a1789df00c3","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:42.787Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e302a81-968e-4f07-89c1-9a1789df00c3"}
{"level":"info","ts":"2026-07-10T10:44:42.787Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8e302a81-968e-4f07-89c1-9a1789df00c3","RayCluster":"counter-5wbjw","State":""}
{"level":"info","ts":"2026-07-10T10:44:44.037Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f15c3ac1-1134-4396-b1b2-1ba4dd112af5","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.037Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f15c3ac1-1134-4396-b1b2-1ba4dd112af5","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:44:44.037Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f15c3ac1-1134-4396-b1b2-1ba4dd112af5","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:44:44.037Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"f15c3ac1-1134-4396-b1b2-1ba4dd112af5","rayCluster":"counter-5wbjw","podIPs":["10.133.0.38","10.132.0.29"]}
{"level":"info","ts":"2026-07-10T10:44:44.796Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63"}
{"level":"info","ts":"2026-07-10T10:44:44.796Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63"}
{"level":"info","ts":"2026-07-10T10:44:44.796Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:44:44.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:44:44.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:44.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:44:44.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:44.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:44.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:44:44.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:44.797Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:44:44.797Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"e7be505f-5fa7-4f19-9362-13b670473e63","name":"counter-5wbjw","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"823c845e-08f5-4c3c-b7d8-b477eceea7af","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1b58741a-2ede-4974-8400-a5b2b333cb15","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1b58741a-2ede-4974-8400-a5b2b333cb15","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1b58741a-2ede-4974-8400-a5b2b333cb15","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"1b58741a-2ede-4974-8400-a5b2b333cb15","rayCluster":"counter-5wbjw","podIPs":["10.132.0.29","10.133.0.38"]}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","head service name":"counter-5wbjw-head-svc","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","head service URL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8"}
{"level":"info","ts":"2026-07-10T10:44:44.809Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","command":["if","!","ray","job","status","--address","http://counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","counter-dtc9p",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-dtc9p","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","--follow","counter-dtc9p"]}
{"level":"info","ts":"2026-07-10T10:44:44.818Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-10T10:44:44.818Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","SubmissionMode":"K8sJobMode","RayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:44:44.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"188a51b2-ed26-4dbd-8708-0740f5e496a8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:44:44.824Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"823c845e-08f5-4c3c-b7d8-b477eceea7af","name":"counter-5wbjw-head"}
{"level":"info","ts":"2026-07-10T10:44:44.832Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dc9f978d-9be7-4d74-b7ca-bf13e0b516d9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:44.832Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dc9f978d-9be7-4d74-b7ca-bf13e0b516d9","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.832Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dc9f978d-9be7-4d74-b7ca-bf13e0b516d9","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:44.832Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dc9f978d-9be7-4d74-b7ca-bf13e0b516d9"}
{"level":"info","ts":"2026-07-10T10:44:44.840Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"823c845e-08f5-4c3c-b7d8-b477eceea7af","name":"counter-5wbjw-workers"}
{"level":"info","ts":"2026-07-10T10:44:44.840Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"823c845e-08f5-4c3c-b7d8-b477eceea7af","cluster":"counter-5wbjw"}
{"level":"error","ts":"2026-07-10T10:44:44.842Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dc9f978d-9be7-4d74-b7ca-bf13e0b516d9","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.842Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dc9f978d-9be7-4d74-b7ca-bf13e0b516d9"}
{"level":"error","ts":"2026-07-10T10:44:44.842Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"dc9f978d-9be7-4d74-b7ca-bf13e0b516d9","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.842Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1065a81d-46f5-4345-918f-0d74baca444b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:44.842Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1065a81d-46f5-4345-918f-0d74baca444b","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.842Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1065a81d-46f5-4345-918f-0d74baca444b","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:44.842Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1065a81d-46f5-4345-918f-0d74baca444b"}
{"level":"error","ts":"2026-07-10T10:44:44.847Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1065a81d-46f5-4345-918f-0d74baca444b","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.847Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1065a81d-46f5-4345-918f-0d74baca444b"}
{"level":"error","ts":"2026-07-10T10:44:44.847Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1065a81d-46f5-4345-918f-0d74baca444b","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"002a2de1-da12-48c9-84bf-dab88e90006d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:44.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"002a2de1-da12-48c9-84bf-dab88e90006d","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"002a2de1-da12-48c9-84bf-dab88e90006d","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:44.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"002a2de1-da12-48c9-84bf-dab88e90006d"}
{"level":"error","ts":"2026-07-10T10:44:44.852Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"002a2de1-da12-48c9-84bf-dab88e90006d","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.852Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"002a2de1-da12-48c9-84bf-dab88e90006d"}
{"level":"error","ts":"2026-07-10T10:44:44.852Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"002a2de1-da12-48c9-84bf-dab88e90006d","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e4fe819-ffde-41fb-a159-9be5296531b3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:44.858Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e4fe819-ffde-41fb-a159-9be5296531b3","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e4fe819-ffde-41fb-a159-9be5296531b3","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:44.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e4fe819-ffde-41fb-a159-9be5296531b3"}
{"level":"error","ts":"2026-07-10T10:44:44.863Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e4fe819-ffde-41fb-a159-9be5296531b3","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.863Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e4fe819-ffde-41fb-a159-9be5296531b3"}
{"level":"error","ts":"2026-07-10T10:44:44.863Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4e4fe819-ffde-41fb-a159-9be5296531b3","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.925Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8dfcbb6c-8bad-4f13-abdc-5c7e335be25e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:44.925Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8dfcbb6c-8bad-4f13-abdc-5c7e335be25e","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:44.925Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8dfcbb6c-8bad-4f13-abdc-5c7e335be25e","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:44.925Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8dfcbb6c-8bad-4f13-abdc-5c7e335be25e"}
{"level":"error","ts":"2026-07-10T10:44:44.930Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8dfcbb6c-8bad-4f13-abdc-5c7e335be25e","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:44.930Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8dfcbb6c-8bad-4f13-abdc-5c7e335be25e"}
{"level":"error","ts":"2026-07-10T10:44:44.930Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"8dfcbb6c-8bad-4f13-abdc-5c7e335be25e","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b8c1dd13-4c3e-466e-b86d-bef401b47919","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:45.011Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b8c1dd13-4c3e-466e-b86d-bef401b47919","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:45.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b8c1dd13-4c3e-466e-b86d-bef401b47919","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:45.025Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b8c1dd13-4c3e-466e-b86d-bef401b47919"}
{"level":"error","ts":"2026-07-10T10:44:45.030Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b8c1dd13-4c3e-466e-b86d-bef401b47919","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.030Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b8c1dd13-4c3e-466e-b86d-bef401b47919"}
{"level":"error","ts":"2026-07-10T10:44:45.030Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b8c1dd13-4c3e-466e-b86d-bef401b47919","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.191Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"249311ac-8141-443b-8f5c-f8ed61305554","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:45.191Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"249311ac-8141-443b-8f5c-f8ed61305554","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:45.191Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"249311ac-8141-443b-8f5c-f8ed61305554","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:45.191Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"249311ac-8141-443b-8f5c-f8ed61305554"}
{"level":"error","ts":"2026-07-10T10:44:45.197Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"249311ac-8141-443b-8f5c-f8ed61305554","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.197Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"249311ac-8141-443b-8f5c-f8ed61305554"}
{"level":"error","ts":"2026-07-10T10:44:45.197Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"249311ac-8141-443b-8f5c-f8ed61305554","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.518Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"085049ef-3a78-498d-8400-e355aaa3af73","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:45.518Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"085049ef-3a78-498d-8400-e355aaa3af73","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:45.518Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"085049ef-3a78-498d-8400-e355aaa3af73","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:45.519Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"085049ef-3a78-498d-8400-e355aaa3af73"}
{"level":"error","ts":"2026-07-10T10:44:45.524Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"085049ef-3a78-498d-8400-e355aaa3af73","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.524Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"085049ef-3a78-498d-8400-e355aaa3af73"}
{"level":"error","ts":"2026-07-10T10:44:45.524Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"085049ef-3a78-498d-8400-e355aaa3af73","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.841Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b73dd29b-6a58-434c-bb72-aec7cb95bb19","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:45.841Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b73dd29b-6a58-434c-bb72-aec7cb95bb19","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:45.841Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b73dd29b-6a58-434c-bb72-aec7cb95bb19","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:45.841Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b73dd29b-6a58-434c-bb72-aec7cb95bb19"}
{"level":"error","ts":"2026-07-10T10:44:45.846Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b73dd29b-6a58-434c-bb72-aec7cb95bb19","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:45.846Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b73dd29b-6a58-434c-bb72-aec7cb95bb19"}
{"level":"error","ts":"2026-07-10T10:44:45.846Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b73dd29b-6a58-434c-bb72-aec7cb95bb19","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:46.165Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3b8cc66b-0996-43c3-9137-887a9bbccb24","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:46.165Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3b8cc66b-0996-43c3-9137-887a9bbccb24","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:46.165Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3b8cc66b-0996-43c3-9137-887a9bbccb24","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:46.165Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3b8cc66b-0996-43c3-9137-887a9bbccb24"}
{"level":"error","ts":"2026-07-10T10:44:46.171Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3b8cc66b-0996-43c3-9137-887a9bbccb24","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:46.171Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3b8cc66b-0996-43c3-9137-887a9bbccb24"}
{"level":"error","ts":"2026-07-10T10:44:46.171Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"3b8cc66b-0996-43c3-9137-887a9bbccb24","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888"}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888"}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","1 head service found":"counter-5wbjw-head-svc"}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","Found 1 head Pod":"counter-5wbjw-head-n5slx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","head Pod":"counter-5wbjw-head-n5slx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5wbjw-head-n5slx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","worker Pod":"counter-5wbjw-small-group-worker-m8qnb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5wbjw-small-group-worker-m8qnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:44:46.809Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"09d1da2d-77f1-439c-85de-973a68b19888","seconds":300}
{"level":"info","ts":"2026-07-10T10:44:48.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ecb3d448-a4c0-4759-a7f9-8426ee366bd8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:48.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ecb3d448-a4c0-4759-a7f9-8426ee366bd8","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:48.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ecb3d448-a4c0-4759-a7f9-8426ee366bd8","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:48.732Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ecb3d448-a4c0-4759-a7f9-8426ee366bd8"}
{"level":"error","ts":"2026-07-10T10:44:48.737Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ecb3d448-a4c0-4759-a7f9-8426ee366bd8","JobId":"counter-dtc9p","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:48.737Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ecb3d448-a4c0-4759-a7f9-8426ee366bd8"}
{"level":"error","ts":"2026-07-10T10:44:48.737Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ecb3d448-a4c0-4759-a7f9-8426ee366bd8","error":"Job counter-dtc9p does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:44:53.857Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1b3a48c3-3f2e-4040-b73a-6c91c194ce77","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:53.857Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1b3a48c3-3f2e-4040-b73a-6c91c194ce77","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:53.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1b3a48c3-3f2e-4040-b73a-6c91c194ce77","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:53.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1b3a48c3-3f2e-4040-b73a-6c91c194ce77"}
{"level":"info","ts":"2026-07-10T10:44:53.864Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1b3a48c3-3f2e-4040-b73a-6c91c194ce77","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:44:53.864Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"1b3a48c3-3f2e-4040-b73a-6c91c194ce77","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:44:53.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f024fbd6-edc6-4391-ba22-4fc5912bf8e1","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:53.879Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f024fbd6-edc6-4391-ba22-4fc5912bf8e1","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:53.879Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f024fbd6-edc6-4391-ba22-4fc5912bf8e1","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:53.879Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f024fbd6-edc6-4391-ba22-4fc5912bf8e1"}
{"level":"info","ts":"2026-07-10T10:44:53.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"f024fbd6-edc6-4391-ba22-4fc5912bf8e1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:44:56.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0948210d-7ef4-4be6-96e9-65904f6338d3","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:56.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0948210d-7ef4-4be6-96e9-65904f6338d3","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:56.878Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0948210d-7ef4-4be6-96e9-65904f6338d3","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:56.879Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0948210d-7ef4-4be6-96e9-65904f6338d3"}
{"level":"info","ts":"2026-07-10T10:44:56.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0948210d-7ef4-4be6-96e9-65904f6338d3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:44:56.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"0948210d-7ef4-4be6-96e9-65904f6338d3","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:44:56.897Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4647831b-dc10-4742-b936-deafa0df7d35","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:56.897Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4647831b-dc10-4742-b936-deafa0df7d35","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:56.897Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4647831b-dc10-4742-b936-deafa0df7d35","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:56.897Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4647831b-dc10-4742-b936-deafa0df7d35"}
{"level":"info","ts":"2026-07-10T10:44:56.900Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"4647831b-dc10-4742-b936-deafa0df7d35","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:44:59.896Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"daff272e-1ad8-4130-80ab-c76dae965491","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:44:59.896Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"daff272e-1ad8-4130-80ab-c76dae965491","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:44:59.896Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"daff272e-1ad8-4130-80ab-c76dae965491","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:44:59.896Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"daff272e-1ad8-4130-80ab-c76dae965491"}
{"level":"info","ts":"2026-07-10T10:44:59.899Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"daff272e-1ad8-4130-80ab-c76dae965491","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:45:02.901Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"5edc990c-813d-4e95-bb62-2027750eea0c","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:02.901Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"5edc990c-813d-4e95-bb62-2027750eea0c","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:02.901Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"5edc990c-813d-4e95-bb62-2027750eea0c","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:02.901Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"5edc990c-813d-4e95-bb62-2027750eea0c"}
{"level":"info","ts":"2026-07-10T10:45:02.905Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"5edc990c-813d-4e95-bb62-2027750eea0c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:45:03.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"2e28db66-1e56-424e-add4-30ab92547062","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:03.911Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"2e28db66-1e56-424e-add4-30ab92547062","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:03.911Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"2e28db66-1e56-424e-add4-30ab92547062","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:03.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"2e28db66-1e56-424e-add4-30ab92547062"}
{"level":"info","ts":"2026-07-10T10:45:03.914Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"2e28db66-1e56-424e-add4-30ab92547062","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:45:04.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"425c937e-3ccf-40ef-915d-a7d57e17ef58","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:04.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"425c937e-3ccf-40ef-915d-a7d57e17ef58","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"425c937e-3ccf-40ef-915d-a7d57e17ef58","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:04.919Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"425c937e-3ccf-40ef-915d-a7d57e17ef58"}
{"level":"info","ts":"2026-07-10T10:45:04.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"425c937e-3ccf-40ef-915d-a7d57e17ef58","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:45:04.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6af282-8c3d-4f57-897b-4898b7d9ce64","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:04.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6af282-8c3d-4f57-897b-4898b7d9ce64","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6af282-8c3d-4f57-897b-4898b7d9ce64","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:04.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6af282-8c3d-4f57-897b-4898b7d9ce64"}
{"level":"info","ts":"2026-07-10T10:45:04.939Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6af282-8c3d-4f57-897b-4898b7d9ce64","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:44:52Z","endTime":"2026-07-10T10:44:56Z"},"jobId":"counter-dtc9p","rayClusterName":"counter-5wbjw","dashboardURL":"counter-5wbjw-head-svc.test-ns-pqpm4.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-10T10:43:13Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:44:44Z","stateTransitionTimes":{"ready":"2026-07-10T10:44:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.29","serviceIP":"10.132.0.29","podName":"counter-5wbjw-head-n5slx","serviceName":"counter-5wbjw-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:43:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:44:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:43:13Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:43:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:45:04.940Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"bf6af282-8c3d-4f57-897b-4898b7d9ce64","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-10T10:45:04.956Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e1808366-c80e-4c45-80e9-3c76d61dad06","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:04.956Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e1808366-c80e-4c45-80e9-3c76d61dad06","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:45:04 +0000 UTC","Now":"2026-07-10T10:45:04.956Z","ShutdownTime":"2026-07-10T10:45:04.000Z"}
{"level":"info","ts":"2026-07-10T10:45:04.956Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e1808366-c80e-4c45-80e9-3c76d61dad06","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:45:04.965Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e1808366-c80e-4c45-80e9-3c76d61dad06","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:04.965Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e1808366-c80e-4c45-80e9-3c76d61dad06","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:45:04.965Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e1808366-c80e-4c45-80e9-3c76d61dad06","RayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"5656a399-bb7a-4c90-bc85-895f5388796d"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"89abb55a-9be6-40bb-9a93-ba1ec87eb314","rayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"reconcileID":"06acdb3e-7985-4886-9a3c-20e56abdd6cd"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"89abb55a-9be6-40bb-9a93-ba1ec87eb314","pod":"counter-5wbjw-head-n5slx","ip":"10.132.0.29"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"89abb55a-9be6-40bb-9a93-ba1ec87eb314","pod":"counter-5wbjw-small-group-worker-m8qnb","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e7f63aef-2b08-403d-a62f-0207a0f4dac4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e7f63aef-2b08-403d-a62f-0207a0f4dac4","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:45:04 +0000 UTC","Now":"2026-07-10T10:45:04.966Z","ShutdownTime":"2026-07-10T10:45:04.000Z"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"89abb55a-9be6-40bb-9a93-ba1ec87eb314","rayCluster":"counter-5wbjw","podIPs":["10.132.0.29","10.133.0.38"]}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e7f63aef-2b08-403d-a62f-0207a0f4dac4","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e7f63aef-2b08-403d-a62f-0207a0f4dac4","RayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e7f63aef-2b08-403d-a62f-0207a0f4dac4","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"e7f63aef-2b08-403d-a62f-0207a0f4dac4","RayCluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:04.966Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","namespace":"test-ns-pqpm4","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:45:04.972Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","name":"kuberay-gateway-access","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:45:04.978Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","name":"test-ns-pqpm4-counter-5wbjw","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:45:04.982Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","configMap":"kube-rbac-proxy-config-counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.025Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","serviceAccount":"counter-5wbjw-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:45:05.047Z","logger":"raycluster-resource","msg":"default","name":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.047Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-5wbjw","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"e36bdf75-d4c9-467f-aba5-fd2f0967cce2","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938","cluster":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938","namespace":"test-ns-pqpm4","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"51f85601-9994-4bb9-b0fd-be11489b6784"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"0c0e4706-6245-4933-a40d-fee2020eee1d","namespace":"test-ns-pqpm4","clusterName":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"c3a365ef-9924-40b0-b322-0018ff7887c8","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"c3a365ef-9924-40b0-b322-0018ff7887c8","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:45:04 +0000 UTC","Now":"2026-07-10T10:45:05.056Z","ShutdownTime":"2026-07-10T10:45:04.000Z"}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"c3a365ef-9924-40b0-b322-0018ff7887c8","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:45:05.056Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"c3a365ef-9924-40b0-b322-0018ff7887c8","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:05.057Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"c3a365ef-9924-40b0-b322-0018ff7887c8","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:45:05.057Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"c3a365ef-9924-40b0-b322-0018ff7887c8","RayCluster":"counter-5wbjw"}
{"level":"error","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938","error":"rayclusters.ray.io \"counter-5wbjw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938"}
{"level":"error","ts":"2026-07-10T10:45:05.125Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"6aadef07-2120-41dc-82e3-b7c445100938","error":"rayclusters.ray.io \"counter-5wbjw\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b8967491-2ce5-4a50-b058-2b49a1a585c9","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b8967491-2ce5-4a50-b058-2b49a1a585c9"}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"8f691bf5-97c6-4b03-9299-12fba8dcf905"}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b8967491-2ce5-4a50-b058-2b49a1a585c9","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"b8967491-2ce5-4a50-b058-2b49a1a585c9","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"21dfc92b-0572-44f6-beba-b6f38a318aac","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"21dfc92b-0572-44f6-beba-b6f38a318aac"}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"21dfc92b-0572-44f6-beba-b6f38a318aac","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:45:05.125Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"21dfc92b-0572-44f6-beba-b6f38a318aac","namespace":"test-ns-pqpm4"}
{"level":"error","ts":"2026-07-10T10:45:05.129Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"0c0e4706-6245-4933-a40d-fee2020eee1d","certificate":"ray-head-cert-counter-5wbjw","error":"certificates.cert-manager.io \"ray-head-cert-counter-5wbjw\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:05.146Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"0c0e4706-6245-4933-a40d-fee2020eee1d","errorCount":1}
{"level":"info","ts":"2026-07-10T10:45:05.146Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"0c0e4706-6245-4933-a40d-fee2020eee1d"}
{"level":"error","ts":"2026-07-10T10:45:05.146Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"0c0e4706-6245-4933-a40d-fee2020eee1d","error":"certificates.cert-manager.io \"ray-head-cert-counter-5wbjw\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:05.151Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"5d66122b-fe0b-4ba6-b275-6fc5e05a410d","namespace":"test-ns-pqpm4","clusterName":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.151Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"5d66122b-fe0b-4ba6-b275-6fc5e05a410d","namespace":"test-ns-pqpm4","clusterName":"counter-5wbjw"}
{"level":"info","ts":"2026-07-10T10:45:05.225Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"24eaf7bb-b749-41cd-a1c9-11c3745eec1b","namespacedName":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"}}
{"level":"info","ts":"2026-07-10T10:45:05.225Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"24eaf7bb-b749-41cd-a1c9-11c3745eec1b"}
{"level":"info","ts":"2026-07-10T10:45:05.225Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"24eaf7bb-b749-41cd-a1c9-11c3745eec1b","namespace":"test-ns-pqpm4","authClusters":0}
{"level":"info","ts":"2026-07-10T10:45:05.225Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5wbjw","namespace":"test-ns-pqpm4"},"namespace":"test-ns-pqpm4","name":"counter-5wbjw","reconcileID":"24eaf7bb-b749-41cd-a1c9-11c3745eec1b","namespace":"test-ns-pqpm4"}
{"level":"info","ts":"2026-07-10T10:45:05.358Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6b8689ca-ae10-401c-9d99-8f763e7fed44","DeletionTimestamp":"2026-07-10 10:45:05 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:45:05.358Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"6b8689ca-ae10-401c-9d99-8f763e7fed44","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:45:05.372Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b328d72e-3167-4d76-9caa-e96fa6cedaad"}
{"level":"info","ts":"2026-07-10T10:45:05.376Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"ba024ff7-8d9f-4682-97a9-42d0af90b428"}
{"level":"info","ts":"2026-07-10T10:45:05.905Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-pqpm4"},"reconcileID":"b6027332-2b43-45e7-a7a8-9c3b9087885b"}
{"level":"info","ts":"2026-07-10T10:45:07.456Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4e05860-4f6a-4041-b098-41f658bf6922","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:07.456Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4e05860-4f6a-4041-b098-41f658bf6922","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:45:07.469Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4e05860-4f6a-4041-b098-41f658bf6922"}
{"level":"info","ts":"2026-07-10T10:45:07.469Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4e05860-4f6a-4041-b098-41f658bf6922","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:45:07.469Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4e05860-4f6a-4041-b098-41f658bf6922","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:45:07.469Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4e05860-4f6a-4041-b098-41f658bf6922","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:45:07.481Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"98f2a97d-4793-4448-9401-37ec408fe798","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:07.481Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"98f2a97d-4793-4448-9401-37ec408fe798","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.481Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"98f2a97d-4793-4448-9401-37ec408fe798","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.481Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"98f2a97d-4793-4448-9401-37ec408fe798","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.487Z","logger":"raycluster-resource","msg":"default","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.487Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-5hvm7","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"98f2a97d-4793-4448-9401-37ec408fe798","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"98f2a97d-4793-4448-9401-37ec408fe798"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"98f2a97d-4793-4448-9401-37ec408fe798","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","namespacedName":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","mode":"IntegratedOAuth","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"74301fee-63b9-4d40-a09a-a30768b93b18","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"cff590f0-04e7-4eee-a72b-89442bc2558e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"cff590f0-04e7-4eee-a72b-89442bc2558e","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"cff590f0-04e7-4eee-a72b-89442bc2558e","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"cff590f0-04e7-4eee-a72b-89442bc2558e"}
{"level":"info","ts":"2026-07-10T10:45:07.503Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"cff590f0-04e7-4eee-a72b-89442bc2558e","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:07.509Z","logger":"raycluster-resource","msg":"default","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.509Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-5hvm7","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:45:07.511Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","name":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:07.511Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:07.511Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.511Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:07.511Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.525Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","serviceSelectors":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/identifier":"counter-5hvm7-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.525Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:45:07.525Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","name":"counter-5hvm7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:07Z","head":{"serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:45:07.525Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"74301fee-63b9-4d40-a09a-a30768b93b18","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.525Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e2cac748-dd50-4e51-89b6-330bac0213b0","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.526Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"0dea3c26-e983-4574-9621-207c128a9a2b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:07.526Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"0dea3c26-e983-4574-9621-207c128a9a2b","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.526Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"0dea3c26-e983-4574-9621-207c128a9a2b","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.526Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"0dea3c26-e983-4574-9621-207c128a9a2b"}
{"level":"info","ts":"2026-07-10T10:45:07.526Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"0dea3c26-e983-4574-9621-207c128a9a2b","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","name":"counter-5hvm7-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","name":"counter-5hvm7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"counter-5hvm7"}}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5"}
{"level":"error","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aa7e3b6c-bfb1-4948-81e8-d58ad12579f5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:45:07.625Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","name":"counter-5hvm7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:45:07.632Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e2cac748-dd50-4e51-89b6-330bac0213b0","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:45:07.632Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg","operation":"created"}
{"level":"info","ts":"2026-07-10T10:45:07.636Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:45:07.636Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.636Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446"}
{"level":"error","ts":"2026-07-10T10:45:07.636Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"41645126-0686-45dc-add8-c99a6fef4446","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"215bd158-ab1e-4aec-b31c-dfddf1eb42eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"215bd158-ab1e-4aec-b31c-dfddf1eb42eb","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"215bd158-ab1e-4aec-b31c-dfddf1eb42eb","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"error","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"215bd158-ab1e-4aec-b31c-dfddf1eb42eb"}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"215bd158-ab1e-4aec-b31c-dfddf1eb42eb","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4"}
{"level":"error","ts":"2026-07-10T10:45:07.637Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dac9b208-fe97-41ce-9a56-bdfa485d11a4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.640Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","name":"test-ns-z4qpg-counter-5hvm7","operation":"created"}
{"level":"info","ts":"2026-07-10T10:45:07.646Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c"}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c"}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c"}
{"level":"error","ts":"2026-07-10T10:45:07.647Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"50a1b5e7-cd69-452b-ae3e-8efaa73dba4c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.649Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e2cac748-dd50-4e51-89b6-330bac0213b0","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:45:07.649Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e2cac748-dd50-4e51-89b6-330bac0213b0","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.649Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","name":"kube-rbac-proxy-config-counter-5hvm7","operation":"created"}
{"level":"info","ts":"2026-07-10T10:45:07.649Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55d5125d-4a19-4faa-8a26-188d22c6142f","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:07.725Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad"}
{"level":"error","ts":"2026-07-10T10:45:07.725Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"86f09014-7469-4d27-abcf-b94fe4f465ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf"}
{"level":"error","ts":"2026-07-10T10:45:07.725Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e53f1c32-6301-49f9-b687-76a24d7828bf","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"8b5e7955-f2bb-4c5e-9d36-427100f34bf8","namespacedName":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"8b5e7955-f2bb-4c5e-9d36-427100f34bf8","mode":"IntegratedOAuth","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.725Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"8b5e7955-f2bb-4c5e-9d36-427100f34bf8","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.729Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55d5125d-4a19-4faa-8a26-188d22c6142f","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:45:07.739Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","rayCluster":"counter-5hvm7","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:45:07.739Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-5hvm7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.739Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b"}
{"level":"error","ts":"2026-07-10T10:45:07.739Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6c023696-c9c7-4305-bdc6-e0cdf533164b","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-5hvm7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.739Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7200836e-e2f4-400b-a8a3-bf40c6289f64","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.740Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"8b5e7955-f2bb-4c5e-9d36-427100f34bf8","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.740Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"08e27c49-f15e-479b-82ff-1a48872dd9a2","namespacedName":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.740Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"08e27c49-f15e-479b-82ff-1a48872dd9a2","mode":"IntegratedOAuth","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.740Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"08e27c49-f15e-479b-82ff-1a48872dd9a2","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.740Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"37f8ad6b-3b1f-4894-8e0f-73b52213c757","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:07.740Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"37f8ad6b-3b1f-4894-8e0f-73b52213c757","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.740Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"37f8ad6b-3b1f-4894-8e0f-73b52213c757","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.741Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"37f8ad6b-3b1f-4894-8e0f-73b52213c757"}
{"level":"info","ts":"2026-07-10T10:45:07.741Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"37f8ad6b-3b1f-4894-8e0f-73b52213c757","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55d5125d-4a19-4faa-8a26-188d22c6142f","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55d5125d-4a19-4faa-8a26-188d22c6142f","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"9648b265-7681-49be-a80d-19fd8c6b2348","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:07.825Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"08e27c49-f15e-479b-82ff-1a48872dd9a2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"08e27c49-f15e-479b-82ff-1a48872dd9a2"}
{"level":"error","ts":"2026-07-10T10:45:07.825Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"08e27c49-f15e-479b-82ff-1a48872dd9a2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:45:07.825Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7200836e-e2f4-400b-a8a3-bf40c6289f64","error":"certificates.cert-manager.io \"ray-worker-cert-counter-5hvm7\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-07-10T10:45:07.825Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7200836e-e2f4-400b-a8a3-bf40c6289f64","error":"certificates.cert-manager.io \"ray-worker-cert-counter-5hvm7\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7200836e-e2f4-400b-a8a3-bf40c6289f64"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-07-10T10:45:07.825Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7200836e-e2f4-400b-a8a3-bf40c6289f64","error":"certificates.cert-manager.io \"ray-worker-cert-counter-5hvm7\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed"}
{"level":"error","ts":"2026-07-10T10:45:07.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"4c366a76-6c51-428b-86d3-6cffa352a9ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"0fdc76ec-8445-4c2e-a267-9053de67e8de","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.825Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"0fdc76ec-8445-4c2e-a267-9053de67e8de"}
{"level":"info","ts":"2026-07-10T10:45:07.830Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4aad67b9-b1cd-43f5-a868-c80ec1e891d0","namespacedName":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:07.830Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4aad67b9-b1cd-43f5-a868-c80ec1e891d0","mode":"IntegratedOAuth","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.830Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4aad67b9-b1cd-43f5-a868-c80ec1e891d0","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.836Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"f9719d3c-3b81-42b7-9760-f29971d83fbb","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.836Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"f9719d3c-3b81-42b7-9760-f29971d83fbb"}
{"level":"info","ts":"2026-07-10T10:45:07.840Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"9648b265-7681-49be-a80d-19fd8c6b2348","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:45:07.842Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4aad67b9-b1cd-43f5-a868-c80ec1e891d0","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.850Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"9648b265-7681-49be-a80d-19fd8c6b2348","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:45:07.850Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"9648b265-7681-49be-a80d-19fd8c6b2348","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0"}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0"}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0"}
{"level":"error","ts":"2026-07-10T10:45:07.986Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"dd89c592-cdbc-430e-8643-175ea4df21a0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986"}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986"}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986"}
{"level":"error","ts":"2026-07-10T10:45:08.307Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e6983b0f-4e57-4121-a03e-1d67c7d39986","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7"}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7"}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","error":"head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"error","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found"}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"counter-5hvm7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7"}
{"level":"error","ts":"2026-07-10T10:45:08.948Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"bb8e86a6-9373-433d-a036-7a7ca0ecaea7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-5hvm7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","mode":"IntegratedOAuth","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","cluster":"counter-5hvm7","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","cluster":"counter-5hvm7","authType":"IntegratedOAuth","serviceAccount":"counter-5hvm7-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-5hvm7","ray.io/group":"headgroup","ray.io/identifier":"counter-5hvm7-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:45:10.229Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","enabled":true}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","name":"counter-5hvm7-head-579gr"}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7"}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","nodeType":"worker","rayStartParams":{"address":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","rayStartCmd":"ray start  --address=counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:45:10.247Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","enabled":true}
{"level":"info","ts":"2026-07-10T10:45:10.261Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","name":"counter-5hvm7-small-group-worker-bl5kq"}
{"level":"info","ts":"2026-07-10T10:45:10.262Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.262Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.262Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-5hvm7-head-svc"},"newHead":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"}}
{"level":"info","ts":"2026-07-10T10:45:10.262Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1714f4ce-a9bd-4de9-be70-3e4e4041dde7","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d"}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d"}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4523ef00-00bf-4422-98ac-25e360ed9c1a","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.278Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e74200dd-17b1-47be-afdc-03612de31f7b","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"7b94cea4-8244-4276-b23e-19a145497208","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"7b94cea4-8244-4276-b23e-19a145497208","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"7b94cea4-8244-4276-b23e-19a145497208","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e74200dd-17b1-47be-afdc-03612de31f7b","rayCluster":"counter-5hvm7","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-5hvm7-head-svc"},"newHead":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"}}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"7b94cea4-8244-4276-b23e-19a145497208"}
{"level":"info","ts":"2026-07-10T10:45:10.279Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"7b94cea4-8244-4276-b23e-19a145497208","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","name":"counter-5hvm7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"counter-5hvm7"}}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d"}
{"level":"error","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"72a35922-3f15-42d7-9a96-251a16f2373d","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.286Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.287Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:45:10.287Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c0177c5c-2b00-4a80-8c59-ce06513f03f3","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:45:10.289Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4523ef00-00bf-4422-98ac-25e360ed9c1a","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4523ef00-00bf-4422-98ac-25e360ed9c1a","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"4523ef00-00bf-4422-98ac-25e360ed9c1a","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"40786b75-d9c9-4a80-8a80-5cfb257b85ee","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"bf648660-9d1f-4462-9543-27fb8c808cda","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:10.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5284d5c6-7021-437a-9c31-65ae75defde3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5284d5c6-7021-437a-9c31-65ae75defde3","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.326Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","podName":"counter-5hvm7-small-group-worker-bl5kq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5284d5c6-7021-437a-9c31-65ae75defde3","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:10.326Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"40786b75-d9c9-4a80-8a80-5cfb257b85ee","rayCluster":"counter-5hvm7","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","podName":"counter-5hvm7-small-group-worker-bl5kq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.339Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"bf648660-9d1f-4462-9543-27fb8c808cda","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5b36cb02-00e6-4e34-8100-f37a13c109ed","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5284d5c6-7021-437a-9c31-65ae75defde3"}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5284d5c6-7021-437a-9c31-65ae75defde3","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5b36cb02-00e6-4e34-8100-f37a13c109ed","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5b36cb02-00e6-4e34-8100-f37a13c109ed","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:45:10.425Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5b36cb02-00e6-4e34-8100-f37a13c109ed","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:45:10.432Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","name":"counter-5hvm7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"counter-5hvm7"}}
{"level":"info","ts":"2026-07-10T10:45:10.433Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514"}
{"level":"error","ts":"2026-07-10T10:45:10.433Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"8330390c-2a89-4301-8c59-2cc6a5f88514","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5hvm7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:45:10.436Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"bf648660-9d1f-4462-9543-27fb8c808cda","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:45:10.437Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"bf648660-9d1f-4462-9543-27fb8c808cda","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:45:10.438Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"00b84348-5717-4737-bd37-217ce45ff06d","seconds":300}
{"level":"info","ts":"2026-07-10T10:45:10.481Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"42894d5c-b0f0-4b8a-a5bd-daf0953b43e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:10.482Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"42894d5c-b0f0-4b8a-a5bd-daf0953b43e3","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:10.482Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"42894d5c-b0f0-4b8a-a5bd-daf0953b43e3","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:10.482Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"42894d5c-b0f0-4b8a-a5bd-daf0953b43e3"}
{"level":"info","ts":"2026-07-10T10:45:10.482Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"42894d5c-b0f0-4b8a-a5bd-daf0953b43e3","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:45:10.712Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"aebc5e2d-0861-4799-8a0e-b6a89c216251","seconds":300}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:45:10.715Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1d00f13f-9fbf-4463-a279-f7bf98cb0647","seconds":300}
{"level":"info","ts":"2026-07-10T10:45:10.950Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47"}
{"level":"info","ts":"2026-07-10T10:45:10.950Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47"}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:45:10.951Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"02b0a728-cf04-4b1f-831f-e482889d1d47","seconds":300}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f"}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f"}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:11.334Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:11.335Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"newHead":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"}}
{"level":"info","ts":"2026-07-10T10:45:11.335Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"23173803-df16-47ec-ac58-c422ed35d95f","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"22ef9d2b-63a6-43d2-9e98-17616b074518","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55ad525a-f113-4c91-848b-9930ddc1b0c2","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"22ef9d2b-63a6-43d2-9e98-17616b074518","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"22ef9d2b-63a6-43d2-9e98-17616b074518","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"60eff353-58c5-4b69-aaa3-322975ded234","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"60eff353-58c5-4b69-aaa3-322975ded234","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"60eff353-58c5-4b69-aaa3-322975ded234","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"60eff353-58c5-4b69-aaa3-322975ded234"}
{"level":"info","ts":"2026-07-10T10:45:11.346Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"60eff353-58c5-4b69-aaa3-322975ded234","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:11.356Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"22ef9d2b-63a6-43d2-9e98-17616b074518","certificate":"ray-head-cert-counter-5hvm7","dnsNames":["counter-5hvm7-head-svc","localhost","counter-5hvm7-head-svc.test-ns-z4qpg.svc","counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.30","10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:45:11.358Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55ad525a-f113-4c91-848b-9930ddc1b0c2","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:45:11.368Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"22ef9d2b-63a6-43d2-9e98-17616b074518","certificate":"ray-worker-cert-counter-5hvm7","dnsNames":["counter-5hvm7-worker-svc","localhost","counter-5hvm7-small-group","counter-5hvm7-worker-svc.test-ns-z4qpg.svc","counter-5hvm7-worker-svc.test-ns-z4qpg.svc.cluster.local","*.counter-5hvm7-worker-svc.test-ns-z4qpg.svc","*.counter-5hvm7-worker-svc.test-ns-z4qpg.svc.cluster.local","*-worker-*.test-ns-z4qpg.svc","*-worker-*.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.30","10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:45:11.368Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"22ef9d2b-63a6-43d2-9e98-17616b074518","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:45:11.371Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55ad525a-f113-4c91-848b-9930ddc1b0c2","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:45:11.371Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"55ad525a-f113-4c91-848b-9930ddc1b0c2","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:45:13.346Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"204c6947-3334-4953-802c-f2cf650ebf7a","seconds":300}
{"level":"info","ts":"2026-07-10T10:45:13.482Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"036cc361-9dc2-4922-9ce8-9ff2038a3eea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:13.482Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"036cc361-9dc2-4922-9ce8-9ff2038a3eea","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:13.482Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"036cc361-9dc2-4922-9ce8-9ff2038a3eea","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:13.482Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"036cc361-9dc2-4922-9ce8-9ff2038a3eea"}
{"level":"info","ts":"2026-07-10T10:45:13.482Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"036cc361-9dc2-4922-9ce8-9ff2038a3eea","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63"}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63"}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:15.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:15.974Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:15.974Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:15.974Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:45:15.974Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"847648ac-932d-4b8a-b913-08cf0d05fd63","seconds":300}
{"level":"info","ts":"2026-07-10T10:45:16.482Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e8d9194c-84e8-430d-8b08-919959af832a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:16.483Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e8d9194c-84e8-430d-8b08-919959af832a","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:16.483Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e8d9194c-84e8-430d-8b08-919959af832a","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:16.483Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e8d9194c-84e8-430d-8b08-919959af832a"}
{"level":"info","ts":"2026-07-10T10:45:16.483Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e8d9194c-84e8-430d-8b08-919959af832a","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:45:16.981Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"6114b990-8a60-4b59-94c2-90c8d928841c","seconds":300}
{"level":"info","ts":"2026-07-10T10:45:19.483Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"db6d43f0-7f56-479f-bf82-3c9960581bb2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:19.483Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"db6d43f0-7f56-479f-bf82-3c9960581bb2","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:19.483Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"db6d43f0-7f56-479f-bf82-3c9960581bb2","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:19.483Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"db6d43f0-7f56-479f-bf82-3c9960581bb2"}
{"level":"info","ts":"2026-07-10T10:45:19.483Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"db6d43f0-7f56-479f-bf82-3c9960581bb2","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:22.484Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"bf5e29ed-f837-472f-87e7-386101e23a13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:22.484Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"bf5e29ed-f837-472f-87e7-386101e23a13","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:22.484Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"bf5e29ed-f837-472f-87e7-386101e23a13","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:22.484Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"bf5e29ed-f837-472f-87e7-386101e23a13"}
{"level":"info","ts":"2026-07-10T10:45:22.484Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"bf5e29ed-f837-472f-87e7-386101e23a13","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:25.485Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"74ca9a9e-ca4a-4510-a58a-8d9ee0943f34","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:25.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"74ca9a9e-ca4a-4510-a58a-8d9ee0943f34","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:25.486Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"74ca9a9e-ca4a-4510-a58a-8d9ee0943f34","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:25.486Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"74ca9a9e-ca4a-4510-a58a-8d9ee0943f34"}
{"level":"info","ts":"2026-07-10T10:45:25.486Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"74ca9a9e-ca4a-4510-a58a-8d9ee0943f34","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:28.486Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fdbeb0e3-f8e2-45bf-8e74-ad5abfa44de6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:28.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fdbeb0e3-f8e2-45bf-8e74-ad5abfa44de6","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:28.486Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fdbeb0e3-f8e2-45bf-8e74-ad5abfa44de6","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:28.487Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fdbeb0e3-f8e2-45bf-8e74-ad5abfa44de6"}
{"level":"info","ts":"2026-07-10T10:45:28.487Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fdbeb0e3-f8e2-45bf-8e74-ad5abfa44de6","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:31.488Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6642ee9d-48ed-4a9d-a7c9-777d6503cf02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:31.488Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6642ee9d-48ed-4a9d-a7c9-777d6503cf02","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:31.488Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6642ee9d-48ed-4a9d-a7c9-777d6503cf02","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:31.488Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6642ee9d-48ed-4a9d-a7c9-777d6503cf02"}
{"level":"info","ts":"2026-07-10T10:45:31.488Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6642ee9d-48ed-4a9d-a7c9-777d6503cf02","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:34.489Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"630f3764-bceb-4404-8e1f-a75434f7fc0a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:34.489Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"630f3764-bceb-4404-8e1f-a75434f7fc0a","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:34.489Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"630f3764-bceb-4404-8e1f-a75434f7fc0a","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:34.489Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"630f3764-bceb-4404-8e1f-a75434f7fc0a"}
{"level":"info","ts":"2026-07-10T10:45:34.489Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"630f3764-bceb-4404-8e1f-a75434f7fc0a","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:37.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8ef399ff-08ff-4f32-87ad-8fdebf8f6270","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:37.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8ef399ff-08ff-4f32-87ad-8fdebf8f6270","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:37.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8ef399ff-08ff-4f32-87ad-8fdebf8f6270","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:37.490Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8ef399ff-08ff-4f32-87ad-8fdebf8f6270"}
{"level":"info","ts":"2026-07-10T10:45:37.490Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8ef399ff-08ff-4f32-87ad-8fdebf8f6270","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:37.836Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"f37db77a-8fb7-4e53-a6bb-70806a4959c0","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:37.837Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"f37db77a-8fb7-4e53-a6bb-70806a4959c0","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:45:37.837Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"f37db77a-8fb7-4e53-a6bb-70806a4959c0","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:45:37.837Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"f37db77a-8fb7-4e53-a6bb-70806a4959c0","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:45:40.491Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6c113530-af13-450e-af30-3df845382667","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:40.491Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6c113530-af13-450e-af30-3df845382667","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:40.491Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6c113530-af13-450e-af30-3df845382667","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:40.492Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6c113530-af13-450e-af30-3df845382667"}
{"level":"info","ts":"2026-07-10T10:45:40.492Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6c113530-af13-450e-af30-3df845382667","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:43.492Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"829fb7e1-7536-4a6d-a381-ca2bbc611054","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:43.492Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"829fb7e1-7536-4a6d-a381-ca2bbc611054","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:43.492Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"829fb7e1-7536-4a6d-a381-ca2bbc611054","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:43.493Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"829fb7e1-7536-4a6d-a381-ca2bbc611054"}
{"level":"info","ts":"2026-07-10T10:45:43.493Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"829fb7e1-7536-4a6d-a381-ca2bbc611054","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:46.494Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"95d656d7-3f1a-447b-8a0a-258f38a220bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:46.494Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"95d656d7-3f1a-447b-8a0a-258f38a220bc","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:46.494Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"95d656d7-3f1a-447b-8a0a-258f38a220bc","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:46.494Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"95d656d7-3f1a-447b-8a0a-258f38a220bc"}
{"level":"info","ts":"2026-07-10T10:45:46.494Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"95d656d7-3f1a-447b-8a0a-258f38a220bc","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:49.495Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8cd077e7-203c-40e5-9711-10c229f4ab75","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:49.495Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8cd077e7-203c-40e5-9711-10c229f4ab75","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:49.495Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8cd077e7-203c-40e5-9711-10c229f4ab75","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:49.495Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8cd077e7-203c-40e5-9711-10c229f4ab75"}
{"level":"info","ts":"2026-07-10T10:45:49.495Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8cd077e7-203c-40e5-9711-10c229f4ab75","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:52.496Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"931308f0-fda3-4e48-aa48-ba3181df1b91","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:52.496Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"931308f0-fda3-4e48-aa48-ba3181df1b91","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:52.496Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"931308f0-fda3-4e48-aa48-ba3181df1b91","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:52.497Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"931308f0-fda3-4e48-aa48-ba3181df1b91"}
{"level":"info","ts":"2026-07-10T10:45:52.497Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"931308f0-fda3-4e48-aa48-ba3181df1b91","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:55.498Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fe415ed6-ebfb-4309-b16a-43d0f4d268e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:55.498Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fe415ed6-ebfb-4309-b16a-43d0f4d268e6","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:55.498Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fe415ed6-ebfb-4309-b16a-43d0f4d268e6","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:55.498Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fe415ed6-ebfb-4309-b16a-43d0f4d268e6"}
{"level":"info","ts":"2026-07-10T10:45:55.498Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fe415ed6-ebfb-4309-b16a-43d0f4d268e6","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:45:58.499Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"70d6be10-9733-4901-a49c-72aae1b04f82","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:45:58.499Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"70d6be10-9733-4901-a49c-72aae1b04f82","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:45:58.499Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"70d6be10-9733-4901-a49c-72aae1b04f82","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:45:58.499Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"70d6be10-9733-4901-a49c-72aae1b04f82"}
{"level":"info","ts":"2026-07-10T10:45:58.499Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"70d6be10-9733-4901-a49c-72aae1b04f82","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:01.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"aeac4e8d-b610-48e8-8e49-046e76391571","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:01.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"aeac4e8d-b610-48e8-8e49-046e76391571","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:01.500Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"aeac4e8d-b610-48e8-8e49-046e76391571","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:01.500Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"aeac4e8d-b610-48e8-8e49-046e76391571"}
{"level":"info","ts":"2026-07-10T10:46:01.500Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"aeac4e8d-b610-48e8-8e49-046e76391571","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:04.501Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8bf4921e-e38d-46ae-95ee-96f548974067","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:04.501Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8bf4921e-e38d-46ae-95ee-96f548974067","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:04.501Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8bf4921e-e38d-46ae-95ee-96f548974067","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:04.501Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8bf4921e-e38d-46ae-95ee-96f548974067"}
{"level":"info","ts":"2026-07-10T10:46:04.501Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8bf4921e-e38d-46ae-95ee-96f548974067","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:07.502Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4e3b3ed4-9fca-466d-aa9c-23eec69f2944","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:07.502Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4e3b3ed4-9fca-466d-aa9c-23eec69f2944","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:07.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4e3b3ed4-9fca-466d-aa9c-23eec69f2944","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:07.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4e3b3ed4-9fca-466d-aa9c-23eec69f2944"}
{"level":"info","ts":"2026-07-10T10:46:07.503Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4e3b3ed4-9fca-466d-aa9c-23eec69f2944","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:10.426Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f7c0610-8ac6-4e69-bddd-debc0c7110c7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:46:10.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f7c0610-8ac6-4e69-bddd-debc0c7110c7","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:46:10.426Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f7c0610-8ac6-4e69-bddd-debc0c7110c7","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:46:10.426Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"6f7c0610-8ac6-4e69-bddd-debc0c7110c7","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:46:10.503Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5fc2969a-b114-4815-ad13-a68873c44759","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:10.503Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5fc2969a-b114-4815-ad13-a68873c44759","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:10.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5fc2969a-b114-4815-ad13-a68873c44759","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:10.504Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5fc2969a-b114-4815-ad13-a68873c44759"}
{"level":"info","ts":"2026-07-10T10:46:10.504Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5fc2969a-b114-4815-ad13-a68873c44759","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:13.504Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9db9b208-dc62-44d9-8424-177107dc5404","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:13.504Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9db9b208-dc62-44d9-8424-177107dc5404","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:13.504Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9db9b208-dc62-44d9-8424-177107dc5404","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:13.505Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9db9b208-dc62-44d9-8424-177107dc5404"}
{"level":"info","ts":"2026-07-10T10:46:13.505Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9db9b208-dc62-44d9-8424-177107dc5404","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:16.505Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9ed974f-50be-4fc4-aa26-059da6c93a09","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:16.505Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9ed974f-50be-4fc4-aa26-059da6c93a09","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:16.505Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9ed974f-50be-4fc4-aa26-059da6c93a09","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:16.505Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9ed974f-50be-4fc4-aa26-059da6c93a09"}
{"level":"info","ts":"2026-07-10T10:46:16.505Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9ed974f-50be-4fc4-aa26-059da6c93a09","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:19.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"62c0d7d8-d987-45b5-adc0-0c06d95fc81e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:19.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"62c0d7d8-d987-45b5-adc0-0c06d95fc81e","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:19.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"62c0d7d8-d987-45b5-adc0-0c06d95fc81e","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:19.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"62c0d7d8-d987-45b5-adc0-0c06d95fc81e"}
{"level":"info","ts":"2026-07-10T10:46:19.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"62c0d7d8-d987-45b5-adc0-0c06d95fc81e","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:22.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1f5d43c0-db5f-4bf3-88eb-bee8ee577dd3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:22.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1f5d43c0-db5f-4bf3-88eb-bee8ee577dd3","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:22.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1f5d43c0-db5f-4bf3-88eb-bee8ee577dd3","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:22.507Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1f5d43c0-db5f-4bf3-88eb-bee8ee577dd3"}
{"level":"info","ts":"2026-07-10T10:46:22.507Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1f5d43c0-db5f-4bf3-88eb-bee8ee577dd3","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:25.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"863f563b-cb14-49ec-a8ba-56901b75378a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:25.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"863f563b-cb14-49ec-a8ba-56901b75378a","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:25.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"863f563b-cb14-49ec-a8ba-56901b75378a","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:25.507Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"863f563b-cb14-49ec-a8ba-56901b75378a"}
{"level":"info","ts":"2026-07-10T10:46:25.507Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"863f563b-cb14-49ec-a8ba-56901b75378a","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:28.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"86b8e678-0e15-48a8-818b-cea68027b2e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:28.508Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"86b8e678-0e15-48a8-818b-cea68027b2e1","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:28.508Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"86b8e678-0e15-48a8-818b-cea68027b2e1","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:28.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"86b8e678-0e15-48a8-818b-cea68027b2e1"}
{"level":"info","ts":"2026-07-10T10:46:28.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"86b8e678-0e15-48a8-818b-cea68027b2e1","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:31.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"871033dd-f53d-4f04-a7a0-77226f3793de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:31.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"871033dd-f53d-4f04-a7a0-77226f3793de","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:31.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"871033dd-f53d-4f04-a7a0-77226f3793de","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:31.509Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"871033dd-f53d-4f04-a7a0-77226f3793de"}
{"level":"info","ts":"2026-07-10T10:46:31.509Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"871033dd-f53d-4f04-a7a0-77226f3793de","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:34.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"210594e5-022d-44dc-b983-97dbe7a2d797","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:34.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"210594e5-022d-44dc-b983-97dbe7a2d797","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:34.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"210594e5-022d-44dc-b983-97dbe7a2d797","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:34.510Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"210594e5-022d-44dc-b983-97dbe7a2d797"}
{"level":"info","ts":"2026-07-10T10:46:34.510Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"210594e5-022d-44dc-b983-97dbe7a2d797","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:36.658Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4"}
{"level":"info","ts":"2026-07-10T10:46:36.658Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4"}
{"level":"info","ts":"2026-07-10T10:46:36.658Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","podName":"counter-5hvm7-head-579gr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:46:36.659Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"a747e630-1977-414b-967d-726da7e774a4","seconds":300}
{"level":"info","ts":"2026-07-10T10:46:37.511Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"061874b2-c02f-44e9-b5cb-cc3e8268251d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:37.511Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"061874b2-c02f-44e9-b5cb-cc3e8268251d","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:37.511Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"061874b2-c02f-44e9-b5cb-cc3e8268251d","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:37.511Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"061874b2-c02f-44e9-b5cb-cc3e8268251d"}
{"level":"info","ts":"2026-07-10T10:46:37.511Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"061874b2-c02f-44e9-b5cb-cc3e8268251d","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412"}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412"}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:46:37.667Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"1f63dc05-7e26-4ae8-8532-69dbbdca4412","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:45:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7965fb35-cdce-4706-bed0-8fba6968b2d1","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e7b9d6fa-2cb5-4dcc-b432-b3aa6da1ee75","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e7b9d6fa-2cb5-4dcc-b432-b3aa6da1ee75","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e7b9d6fa-2cb5-4dcc-b432-b3aa6da1ee75","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"e7b9d6fa-2cb5-4dcc-b432-b3aa6da1ee75","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5aae1fba-13bb-4388-8e94-fd633fa93aef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5aae1fba-13bb-4388-8e94-fd633fa93aef","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5aae1fba-13bb-4388-8e94-fd633fa93aef","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5aae1fba-13bb-4388-8e94-fd633fa93aef"}
{"level":"info","ts":"2026-07-10T10:46:37.680Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"5aae1fba-13bb-4388-8e94-fd633fa93aef","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:37.697Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7965fb35-cdce-4706-bed0-8fba6968b2d1","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:46:37.710Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7965fb35-cdce-4706-bed0-8fba6968b2d1","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:46:37.710Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7965fb35-cdce-4706-bed0-8fba6968b2d1","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:37.837Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"b2c620a9-73e2-4c47-ac7d-e34b4f7620b1","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:37.837Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"b2c620a9-73e2-4c47-ac7d-e34b4f7620b1","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:46:37.837Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"b2c620a9-73e2-4c47-ac7d-e34b4f7620b1","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:46:37.837Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"b2c620a9-73e2-4c47-ac7d-e34b4f7620b1","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1"}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1"}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:46:39.681Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"516e3e91-1dfd-4b21-a7a5-c4882a25a1c1","seconds":300}
{"level":"info","ts":"2026-07-10T10:46:40.511Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d13af8c4-5d8c-4241-b40e-d2d52dc5e1f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:40.511Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d13af8c4-5d8c-4241-b40e-d2d52dc5e1f5","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:40.511Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d13af8c4-5d8c-4241-b40e-d2d52dc5e1f5","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:40.512Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d13af8c4-5d8c-4241-b40e-d2d52dc5e1f5"}
{"level":"info","ts":"2026-07-10T10:46:40.512Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d13af8c4-5d8c-4241-b40e-d2d52dc5e1f5","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c"}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c"}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:46:42.372Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"c1020a42-4140-459a-8fd2-26c2cdc2198c","seconds":300}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf"}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf"}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:46:43.383Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"75db86b4-1c45-4c42-bc65-448ab52b2dbf","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"d7d25f5e-11a8-4d78-bb6c-6fee7c5a13ab","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"514b4cab-df92-4d1e-9b97-7a707692f692","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d5928d77-163d-4595-9bbc-76ebe72fa186","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d5928d77-163d-4595-9bbc-76ebe72fa186","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d5928d77-163d-4595-9bbc-76ebe72fa186","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"514b4cab-df92-4d1e-9b97-7a707692f692","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"514b4cab-df92-4d1e-9b97-7a707692f692","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"514b4cab-df92-4d1e-9b97-7a707692f692","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d5928d77-163d-4595-9bbc-76ebe72fa186"}
{"level":"info","ts":"2026-07-10T10:46:43.397Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"d5928d77-163d-4595-9bbc-76ebe72fa186","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:43.410Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"d7d25f5e-11a8-4d78-bb6c-6fee7c5a13ab","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:46:43.422Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"d7d25f5e-11a8-4d78-bb6c-6fee7c5a13ab","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:46:43.422Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"d7d25f5e-11a8-4d78-bb6c-6fee7c5a13ab","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:43.513Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29007812-b671-43bd-b1cf-be7747861eda","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:43.513Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29007812-b671-43bd-b1cf-be7747861eda","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:43.513Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29007812-b671-43bd-b1cf-be7747861eda","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:43.513Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29007812-b671-43bd-b1cf-be7747861eda"}
{"level":"info","ts":"2026-07-10T10:46:43.513Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29007812-b671-43bd-b1cf-be7747861eda","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:45.397Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1"}
{"level":"info","ts":"2026-07-10T10:46:45.397Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1"}
{"level":"info","ts":"2026-07-10T10:46:45.397Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:45.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:45.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:45.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","podName":"counter-5hvm7-head-579gr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:46:45.398Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"ff4b1a71-46de-441e-b8f7-90d526e12bd1","seconds":300}
{"level":"info","ts":"2026-07-10T10:46:46.514Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4ef9805-bf4d-4767-a642-c154a35e0a5f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:46.514Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4ef9805-bf4d-4767-a642-c154a35e0a5f","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:46.514Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4ef9805-bf4d-4767-a642-c154a35e0a5f","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:46.514Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4ef9805-bf4d-4767-a642-c154a35e0a5f"}
{"level":"info","ts":"2026-07-10T10:46:46.514Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a4ef9805-bf4d-4767-a642-c154a35e0a5f","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:48.696Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100"}
{"level":"info","ts":"2026-07-10T10:46:48.696Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100"}
{"level":"info","ts":"2026-07-10T10:46:48.696Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","podName":"counter-5hvm7-small-group-worker-bl5kq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","podName":"counter-5hvm7-small-group-worker-bl5kq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:46:48.697Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"2640bc8f-9511-4089-aff1-5be3591ff100","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:46:48.709Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"a305c2ea-72f1-4087-a8f9-a1068e1c7482","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:48.709Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"13152c45-e3c4-4b21-8011-ddad7fe85f11","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:48.709Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"c8b339cd-17d4-4703-88ea-61c2780621ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:48.709Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"c8b339cd-17d4-4703-88ea-61c2780621ee","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:48.709Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"c8b339cd-17d4-4703-88ea-61c2780621ee","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:48.710Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"13152c45-e3c4-4b21-8011-ddad7fe85f11","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:46:48.710Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"13152c45-e3c4-4b21-8011-ddad7fe85f11","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:46:48.710Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"13152c45-e3c4-4b21-8011-ddad7fe85f11","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:46:48.710Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"c8b339cd-17d4-4703-88ea-61c2780621ee"}
{"level":"info","ts":"2026-07-10T10:46:48.710Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"c8b339cd-17d4-4703-88ea-61c2780621ee","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:48.723Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"a305c2ea-72f1-4087-a8f9-a1068e1c7482","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:46:48.733Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"a305c2ea-72f1-4087-a8f9-a1068e1c7482","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:46:48.733Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"a305c2ea-72f1-4087-a8f9-a1068e1c7482","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:49.516Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8fc3f1bb-c453-4b83-a3dd-e5bbbc3e49ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:49.516Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8fc3f1bb-c453-4b83-a3dd-e5bbbc3e49ee","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:49.516Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8fc3f1bb-c453-4b83-a3dd-e5bbbc3e49ee","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:49.516Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8fc3f1bb-c453-4b83-a3dd-e5bbbc3e49ee"}
{"level":"info","ts":"2026-07-10T10:46:49.516Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8fc3f1bb-c453-4b83-a3dd-e5bbbc3e49ee","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412"}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412"}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","podName":"counter-5hvm7-small-group-worker-bl5kq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:46:50.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","podName":"counter-5hvm7-small-group-worker-bl5kq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:45:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:46:50.711Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:46:50.711Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"433e0768-6d0c-4f46-8e8c-7fbacd92c412","seconds":300}
{"level":"info","ts":"2026-07-10T10:46:52.517Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6ae6c580-df80-460a-8c81-5c662bb579ca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:52.517Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6ae6c580-df80-460a-8c81-5c662bb579ca","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:52.517Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6ae6c580-df80-460a-8c81-5c662bb579ca","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:52.517Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6ae6c580-df80-460a-8c81-5c662bb579ca"}
{"level":"info","ts":"2026-07-10T10:46:52.517Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6ae6c580-df80-460a-8c81-5c662bb579ca","RayCluster":"counter-5hvm7","State":""}
{"level":"info","ts":"2026-07-10T10:46:54.407Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e"}
{"level":"info","ts":"2026-07-10T10:46:54.407Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e"}
{"level":"info","ts":"2026-07-10T10:46:54.407Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:54.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:46:54.408Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"3d4cdef7-8c2a-4514-9f37-2a1d9cd93c1e","name":"counter-5hvm7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"86288142-dd76-4691-821e-118ae401a2aa","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"c7d201e8-f349-4bf1-a6fd-dfe1563009f1","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"c7d201e8-f349-4bf1-a6fd-dfe1563009f1","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"c7d201e8-f349-4bf1-a6fd-dfe1563009f1","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"c7d201e8-f349-4bf1-a6fd-dfe1563009f1","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","head service name":"counter-5hvm7-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","head service URL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d"}
{"level":"info","ts":"2026-07-10T10:46:54.423Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","command":["if","!","ray","job","status","--address","http://counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","counter-2j89f",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-2j89f","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","--follow","counter-2j89f"]}
{"level":"info","ts":"2026-07-10T10:46:54.432Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-10T10:46:54.432Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","SubmissionMode":"K8sJobMode","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.432Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:46:54.432Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e257068a-5977-451d-b205-9c79bedef52d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:46:54.439Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"86288142-dd76-4691-821e-118ae401a2aa","name":"counter-5hvm7-head"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","head service name":"counter-5hvm7-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","head service URL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","SubmissionMode":"K8sJobMode","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:46:54.446Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:46:54.452Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"86288142-dd76-4691-821e-118ae401a2aa","name":"counter-5hvm7-workers"}
{"level":"info","ts":"2026-07-10T10:46:54.452Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"86288142-dd76-4691-821e-118ae401a2aa","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.453Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:46:54.453Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558"}
{"level":"error","ts":"2026-07-10T10:46:54.453Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"ef0738cf-aebb-4083-b80a-31194024c558","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.453Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29e4e872-6f8d-44a8-b852-b1ef906f9d69","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.453Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29e4e872-6f8d-44a8-b852-b1ef906f9d69","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.453Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29e4e872-6f8d-44a8-b852-b1ef906f9d69","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.453Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29e4e872-6f8d-44a8-b852-b1ef906f9d69"}
{"level":"error","ts":"2026-07-10T10:46:54.463Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29e4e872-6f8d-44a8-b852-b1ef906f9d69","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.464Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29e4e872-6f8d-44a8-b852-b1ef906f9d69"}
{"level":"error","ts":"2026-07-10T10:46:54.464Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"29e4e872-6f8d-44a8-b852-b1ef906f9d69","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.464Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"123a0dda-77f3-4afe-b0af-4853a742c3ec","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.464Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"123a0dda-77f3-4afe-b0af-4853a742c3ec","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.464Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"123a0dda-77f3-4afe-b0af-4853a742c3ec","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.464Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"123a0dda-77f3-4afe-b0af-4853a742c3ec"}
{"level":"error","ts":"2026-07-10T10:46:54.468Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"123a0dda-77f3-4afe-b0af-4853a742c3ec","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.469Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"123a0dda-77f3-4afe-b0af-4853a742c3ec"}
{"level":"error","ts":"2026-07-10T10:46:54.469Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"123a0dda-77f3-4afe-b0af-4853a742c3ec","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.474Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"683e2309-fb04-49ab-bb7e-fa827d3c0f5d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.474Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"683e2309-fb04-49ab-bb7e-fa827d3c0f5d","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.474Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"683e2309-fb04-49ab-bb7e-fa827d3c0f5d","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.474Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"683e2309-fb04-49ab-bb7e-fa827d3c0f5d"}
{"level":"error","ts":"2026-07-10T10:46:54.479Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"683e2309-fb04-49ab-bb7e-fa827d3c0f5d","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.479Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"683e2309-fb04-49ab-bb7e-fa827d3c0f5d"}
{"level":"error","ts":"2026-07-10T10:46:54.479Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"683e2309-fb04-49ab-bb7e-fa827d3c0f5d","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"21514747-505c-46ba-ad55-24b2230e2f02","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"21514747-505c-46ba-ad55-24b2230e2f02","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"21514747-505c-46ba-ad55-24b2230e2f02","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"21514747-505c-46ba-ad55-24b2230e2f02"}
{"level":"error","ts":"2026-07-10T10:46:54.530Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"21514747-505c-46ba-ad55-24b2230e2f02","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.530Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"21514747-505c-46ba-ad55-24b2230e2f02"}
{"level":"error","ts":"2026-07-10T10:46:54.530Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"21514747-505c-46ba-ad55-24b2230e2f02","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.610Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"afdd371b-25a1-4ce2-9b8d-cc84e66d4712","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.610Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"afdd371b-25a1-4ce2-9b8d-cc84e66d4712","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.610Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"afdd371b-25a1-4ce2-9b8d-cc84e66d4712","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.610Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"afdd371b-25a1-4ce2-9b8d-cc84e66d4712"}
{"level":"error","ts":"2026-07-10T10:46:54.616Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"afdd371b-25a1-4ce2-9b8d-cc84e66d4712","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.616Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"afdd371b-25a1-4ce2-9b8d-cc84e66d4712"}
{"level":"error","ts":"2026-07-10T10:46:54.616Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"afdd371b-25a1-4ce2-9b8d-cc84e66d4712","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.777Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e1dad2c7-29cb-49c5-b5a5-ecef666e2a1b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:54.777Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e1dad2c7-29cb-49c5-b5a5-ecef666e2a1b","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:54.777Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e1dad2c7-29cb-49c5-b5a5-ecef666e2a1b","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:54.777Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e1dad2c7-29cb-49c5-b5a5-ecef666e2a1b"}
{"level":"error","ts":"2026-07-10T10:46:54.783Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e1dad2c7-29cb-49c5-b5a5-ecef666e2a1b","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:54.783Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e1dad2c7-29cb-49c5-b5a5-ecef666e2a1b"}
{"level":"error","ts":"2026-07-10T10:46:54.783Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e1dad2c7-29cb-49c5-b5a5-ecef666e2a1b","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:55.103Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8d59de77-02fe-412e-97e8-6296aa7d9922","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:55.103Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8d59de77-02fe-412e-97e8-6296aa7d9922","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:55.103Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8d59de77-02fe-412e-97e8-6296aa7d9922","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:55.103Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8d59de77-02fe-412e-97e8-6296aa7d9922"}
{"level":"error","ts":"2026-07-10T10:46:55.109Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8d59de77-02fe-412e-97e8-6296aa7d9922","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:55.109Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8d59de77-02fe-412e-97e8-6296aa7d9922"}
{"level":"error","ts":"2026-07-10T10:46:55.109Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"8d59de77-02fe-412e-97e8-6296aa7d9922","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:55.461Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"dff3e5bb-5c9e-44d5-b5ce-2061aab89b77","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:55.461Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"dff3e5bb-5c9e-44d5-b5ce-2061aab89b77","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:55.461Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"dff3e5bb-5c9e-44d5-b5ce-2061aab89b77","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:55.461Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"dff3e5bb-5c9e-44d5-b5ce-2061aab89b77"}
{"level":"error","ts":"2026-07-10T10:46:55.469Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"dff3e5bb-5c9e-44d5-b5ce-2061aab89b77","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:55.469Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"dff3e5bb-5c9e-44d5-b5ce-2061aab89b77"}
{"level":"error","ts":"2026-07-10T10:46:55.469Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"dff3e5bb-5c9e-44d5-b5ce-2061aab89b77","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:55.750Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a092efb6-e5ec-4a8d-a147-de5df05168a7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:55.750Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a092efb6-e5ec-4a8d-a147-de5df05168a7","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:55.750Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a092efb6-e5ec-4a8d-a147-de5df05168a7","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:55.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a092efb6-e5ec-4a8d-a147-de5df05168a7"}
{"level":"error","ts":"2026-07-10T10:46:55.756Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a092efb6-e5ec-4a8d-a147-de5df05168a7","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:55.756Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a092efb6-e5ec-4a8d-a147-de5df05168a7"}
{"level":"error","ts":"2026-07-10T10:46:55.756Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"a092efb6-e5ec-4a8d-a147-de5df05168a7","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43"}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43"}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","1 head service found":"counter-5hvm7-head-svc"}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","Found 1 head Pod":"counter-5hvm7-head-579gr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","head Pod":"counter-5hvm7-head-579gr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5hvm7-head-579gr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","worker Pod":"counter-5hvm7-small-group-worker-bl5kq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-5hvm7-small-group-worker-bl5kq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:46:56.423Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"f51bfe19-fa53-4598-b760-f110773eff43","seconds":300}
{"level":"info","ts":"2026-07-10T10:46:58.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6518c39b-b585-4958-a6b5-bb54809fa1bc","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:46:58.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6518c39b-b585-4958-a6b5-bb54809fa1bc","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:46:58.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6518c39b-b585-4958-a6b5-bb54809fa1bc","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:46:58.317Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6518c39b-b585-4958-a6b5-bb54809fa1bc"}
{"level":"error","ts":"2026-07-10T10:46:58.322Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6518c39b-b585-4958-a6b5-bb54809fa1bc","JobId":"counter-2j89f","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:46:58.322Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6518c39b-b585-4958-a6b5-bb54809fa1bc"}
{"level":"error","ts":"2026-07-10T10:46:58.322Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6518c39b-b585-4958-a6b5-bb54809fa1bc","error":"Job counter-2j89f does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:03.443Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0113d3c-4aa2-4ce0-b8c2-4b315d1a6fac","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:03.443Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0113d3c-4aa2-4ce0-b8c2-4b315d1a6fac","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:03.443Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0113d3c-4aa2-4ce0-b8c2-4b315d1a6fac","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:03.443Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0113d3c-4aa2-4ce0-b8c2-4b315d1a6fac"}
{"level":"info","ts":"2026-07-10T10:47:03.450Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0113d3c-4aa2-4ce0-b8c2-4b315d1a6fac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:03.450Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0113d3c-4aa2-4ce0-b8c2-4b315d1a6fac","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:47:03.469Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"80b1c7e0-25ef-4528-9092-5eb974604bfc","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:03.469Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"80b1c7e0-25ef-4528-9092-5eb974604bfc","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:03.469Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"80b1c7e0-25ef-4528-9092-5eb974604bfc","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:03.469Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"80b1c7e0-25ef-4528-9092-5eb974604bfc"}
{"level":"info","ts":"2026-07-10T10:47:03.472Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"80b1c7e0-25ef-4528-9092-5eb974604bfc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:06.468Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fae53f3f-3e8b-40f4-b69f-f5231e0b8ebe","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:06.468Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fae53f3f-3e8b-40f4-b69f-f5231e0b8ebe","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:06.468Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fae53f3f-3e8b-40f4-b69f-f5231e0b8ebe","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:06.468Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fae53f3f-3e8b-40f4-b69f-f5231e0b8ebe"}
{"level":"info","ts":"2026-07-10T10:47:06.472Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fae53f3f-3e8b-40f4-b69f-f5231e0b8ebe","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:06.472Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"fae53f3f-3e8b-40f4-b69f-f5231e0b8ebe","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:47:06.487Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"63f0bdff-7196-45f0-a9e4-84e28825d473","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:06.487Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"63f0bdff-7196-45f0-a9e4-84e28825d473","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:06.487Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"63f0bdff-7196-45f0-a9e4-84e28825d473","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:06.487Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"63f0bdff-7196-45f0-a9e4-84e28825d473"}
{"level":"info","ts":"2026-07-10T10:47:06.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"63f0bdff-7196-45f0-a9e4-84e28825d473","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:09.488Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0b54d70-2618-413b-ab1f-2a63e551f17d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:09.488Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0b54d70-2618-413b-ab1f-2a63e551f17d","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:09.488Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0b54d70-2618-413b-ab1f-2a63e551f17d","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:09.488Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0b54d70-2618-413b-ab1f-2a63e551f17d"}
{"level":"info","ts":"2026-07-10T10:47:09.492Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e0b54d70-2618-413b-ab1f-2a63e551f17d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:10.427Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"65a09acc-6be0-4aa0-8ddc-9307e8f6d670","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:47:10.427Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"65a09acc-6be0-4aa0-8ddc-9307e8f6d670","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:47:10.427Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"65a09acc-6be0-4aa0-8ddc-9307e8f6d670","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:47:10.427Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"65a09acc-6be0-4aa0-8ddc-9307e8f6d670","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:47:12.492Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"285ed0d1-1219-44cc-967f-32e825c1697d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:12.492Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"285ed0d1-1219-44cc-967f-32e825c1697d","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:12.492Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"285ed0d1-1219-44cc-967f-32e825c1697d","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:12.492Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"285ed0d1-1219-44cc-967f-32e825c1697d"}
{"level":"info","ts":"2026-07-10T10:47:12.496Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"285ed0d1-1219-44cc-967f-32e825c1697d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:13.531Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1646f30a-696d-4b99-b30b-c6af689f847e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:13.531Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1646f30a-696d-4b99-b30b-c6af689f847e","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:13.531Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1646f30a-696d-4b99-b30b-c6af689f847e","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:13.531Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1646f30a-696d-4b99-b30b-c6af689f847e"}
{"level":"info","ts":"2026-07-10T10:47:13.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1646f30a-696d-4b99-b30b-c6af689f847e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:14.539Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"f392459e-a1f1-412c-86bc-961912213a87","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:14.539Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"f392459e-a1f1-412c-86bc-961912213a87","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.539Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"f392459e-a1f1-412c-86bc-961912213a87","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:14.539Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"f392459e-a1f1-412c-86bc-961912213a87"}
{"level":"info","ts":"2026-07-10T10:47:14.543Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"f392459e-a1f1-412c-86bc-961912213a87","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:14.558Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9d2bdf33-62ad-4b90-9902-60bb044faf57","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:14.558Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9d2bdf33-62ad-4b90-9902-60bb044faf57","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.558Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9d2bdf33-62ad-4b90-9902-60bb044faf57","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:14.559Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9d2bdf33-62ad-4b90-9902-60bb044faf57"}
{"level":"info","ts":"2026-07-10T10:47:14.562Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9d2bdf33-62ad-4b90-9902-60bb044faf57","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:47:02Z","endTime":"2026-07-10T10:47:06Z"},"jobId":"counter-2j89f","rayClusterName":"counter-5hvm7","dashboardURL":"counter-5hvm7-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-10T10:45:07Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:46:54Z","stateTransitionTimes":{"ready":"2026-07-10T10:46:54Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.30","serviceIP":"10.132.0.30","podName":"counter-5hvm7-head-579gr","serviceName":"counter-5hvm7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:46:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:46:54Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:45:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:45:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:47:14.562Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"9d2bdf33-62ad-4b90-9902-60bb044faf57","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-10T10:47:14.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1da36bbf-e5c8-49fd-b7e8-66816ff66b97","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:14.579Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1da36bbf-e5c8-49fd-b7e8-66816ff66b97","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:47:14 +0000 UTC","Now":"2026-07-10T10:47:14.579Z","ShutdownTime":"2026-07-10T10:47:14.000Z"}
{"level":"info","ts":"2026-07-10T10:47:14.579Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1da36bbf-e5c8-49fd-b7e8-66816ff66b97","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1da36bbf-e5c8-49fd-b7e8-66816ff66b97","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1da36bbf-e5c8-49fd-b7e8-66816ff66b97","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"1da36bbf-e5c8-49fd-b7e8-66816ff66b97","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"70667861-b9f3-40e6-af4b-d3bac9435bba","rayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","namespacedName":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"ed42e39b-3c68-4d0d-a148-febd2d1a682f"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"70667861-b9f3-40e6-af4b-d3bac9435bba","pod":"counter-5hvm7-head-579gr","ip":"10.132.0.30"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"70667861-b9f3-40e6-af4b-d3bac9435bba","pod":"counter-5hvm7-small-group-worker-bl5kq","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"reconcileID":"e2e99b2a-6279-4e1f-8d8e-b2cf9e41abe1"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:47:14.590Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"70667861-b9f3-40e6-af4b-d3bac9435bba","rayCluster":"counter-5hvm7","podIPs":["10.132.0.30","10.133.0.40"]}
{"level":"info","ts":"2026-07-10T10:47:14.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4d1ffe53-4303-4ea3-996a-263fb85cdd48","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:14.591Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4d1ffe53-4303-4ea3-996a-263fb85cdd48","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:47:14 +0000 UTC","Now":"2026-07-10T10:47:14.591Z","ShutdownTime":"2026-07-10T10:47:14.000Z"}
{"level":"info","ts":"2026-07-10T10:47:14.591Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4d1ffe53-4303-4ea3-996a-263fb85cdd48","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:47:14.591Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4d1ffe53-4303-4ea3-996a-263fb85cdd48","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.591Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4d1ffe53-4303-4ea3-996a-263fb85cdd48","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:47:14.591Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"4d1ffe53-4303-4ea3-996a-263fb85cdd48","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.597Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:47:14.602Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","name":"test-ns-z4qpg-counter-5hvm7","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:47:14.608Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","configMap":"kube-rbac-proxy-config-counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.615Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","serviceAccount":"counter-5hvm7-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:47:14.639Z","logger":"raycluster-resource","msg":"default","name":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.640Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-5hvm7","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:47:14.652Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"009dda12-137f-47d7-aa18-b5db15a06f12"}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6bbd34f7-42ff-46a5-9495-372936d12689","namespace":"test-ns-z4qpg","clusterName":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e100068a-7b2e-42e4-a974-12c7fbb68339","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e100068a-7b2e-42e4-a974-12c7fbb68339","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:47:14 +0000 UTC","Now":"2026-07-10T10:47:14.653Z","ShutdownTime":"2026-07-10T10:47:14.000Z"}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e100068a-7b2e-42e4-a974-12c7fbb68339","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e100068a-7b2e-42e4-a974-12c7fbb68339","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e100068a-7b2e-42e4-a974-12c7fbb68339","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e100068a-7b2e-42e4-a974-12c7fbb68339","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"15f85a00-b63f-4a2c-85a4-fad9be068882","cluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"56466027-a6d3-4c8f-b0f0-1c734337a43b","namespacedName":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"56466027-a6d3-4c8f-b0f0-1c734337a43b"}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"56466027-a6d3-4c8f-b0f0-1c734337a43b","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:47:14.653Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"56466027-a6d3-4c8f-b0f0-1c734337a43b","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:47:14.660Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"cd0beb94-0338-4f42-94d1-ddd068f94827"}
{"level":"error","ts":"2026-07-10T10:47:14.725Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6bbd34f7-42ff-46a5-9495-372936d12689","issuer":"ray-selfsigned-issuer-counter-5hvm7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-5hvm7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:14.725Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"00eac981-6927-4240-a920-3bb4ae06aee6","namespacedName":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:14.725Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"00eac981-6927-4240-a920-3bb4ae06aee6"}
{"level":"info","ts":"2026-07-10T10:47:14.725Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"7a4b1295-c489-4b00-9be5-9243b3b14f44"}
{"level":"info","ts":"2026-07-10T10:47:14.725Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"00eac981-6927-4240-a920-3bb4ae06aee6","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:47:14.725Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"00eac981-6927-4240-a920-3bb4ae06aee6","namespace":"test-ns-z4qpg"}
{"level":"error","ts":"2026-07-10T10:47:14.728Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6bbd34f7-42ff-46a5-9495-372936d12689","certificate":"ray-worker-cert-counter-5hvm7","error":"certificates.cert-manager.io \"ray-worker-cert-counter-5hvm7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:14.743Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6bbd34f7-42ff-46a5-9495-372936d12689","errorCount":2}
{"level":"info","ts":"2026-07-10T10:47:14.743Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6bbd34f7-42ff-46a5-9495-372936d12689"}
{"level":"error","ts":"2026-07-10T10:47:14.743Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"6bbd34f7-42ff-46a5-9495-372936d12689","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-5hvm7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:14.748Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"c7165366-ef4a-4963-9b20-2b0e772a7ac0","namespace":"test-ns-z4qpg","clusterName":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:14.748Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"counter-5hvm7","reconcileID":"c7165366-ef4a-4963-9b20-2b0e772a7ac0","namespace":"test-ns-z4qpg","clusterName":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:15.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9fa5dd3-b5bc-4777-91ec-34d812f572ef","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:15.242Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9fa5dd3-b5bc-4777-91ec-34d812f572ef","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:47:14 +0000 UTC","Now":"2026-07-10T10:47:15.242Z","ShutdownTime":"2026-07-10T10:47:14.000Z"}
{"level":"info","ts":"2026-07-10T10:47:15.242Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9fa5dd3-b5bc-4777-91ec-34d812f572ef","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:47:15.242Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9fa5dd3-b5bc-4777-91ec-34d812f572ef","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:15.242Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9fa5dd3-b5bc-4777-91ec-34d812f572ef","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:47:15.242Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"e9fa5dd3-b5bc-4777-91ec-34d812f572ef","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:15.497Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"251594d6-b5ff-44ab-a80f-23600134d93d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:15.497Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"251594d6-b5ff-44ab-a80f-23600134d93d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:47:14 +0000 UTC","Now":"2026-07-10T10:47:15.497Z","ShutdownTime":"2026-07-10T10:47:14.000Z"}
{"level":"info","ts":"2026-07-10T10:47:15.497Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"251594d6-b5ff-44ab-a80f-23600134d93d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:47:15.497Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"251594d6-b5ff-44ab-a80f-23600134d93d","RayCluster":{"name":"counter-5hvm7","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:15.497Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"251594d6-b5ff-44ab-a80f-23600134d93d","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:47:15.497Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"251594d6-b5ff-44ab-a80f-23600134d93d","RayCluster":"counter-5hvm7"}
{"level":"info","ts":"2026-07-10T10:47:45.260Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6366eab5-85a1-4a51-948d-372ef489e738","DeletionTimestamp":"2026-07-10 10:47:45 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:47:45.260Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"6366eab5-85a1-4a51-948d-372ef489e738","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:47:45.275Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b7e97e56-9617-4747-8ea5-c7ce780dcbc1","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.275Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b7e97e56-9617-4747-8ea5-c7ce780dcbc1","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:47:45.288Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b7e97e56-9617-4747-8ea5-c7ce780dcbc1"}
{"level":"info","ts":"2026-07-10T10:47:45.288Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b7e97e56-9617-4747-8ea5-c7ce780dcbc1","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:47:45.288Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b7e97e56-9617-4747-8ea5-c7ce780dcbc1","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:47:45.288Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b7e97e56-9617-4747-8ea5-c7ce780dcbc1","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:47:45.299Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"acbc341e-143d-47b1-938c-868879de2645"}
{"level":"info","ts":"2026-07-10T10:47:45.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.299Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96"}
{"level":"info","ts":"2026-07-10T10:47:45.299Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:47:45.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-cnmvq","rayClusterName":"fail-hpc2w","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:47:45.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:47:45.310Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:47:45.310Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96"}
{"level":"error","ts":"2026-07-10T10:47:45.310Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"72285f64-03d1-4226-929e-d67c4feb6e96","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bf224df7-60f8-4803-be18-300d960fcc42","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bf224df7-60f8-4803-be18-300d960fcc42","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.310Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bf224df7-60f8-4803-be18-300d960fcc42","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.310Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bf224df7-60f8-4803-be18-300d960fcc42","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.315Z","logger":"raycluster-resource","msg":"default","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.315Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-4h4dv","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bf224df7-60f8-4803-be18-300d960fcc42","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bf224df7-60f8-4803-be18-300d960fcc42"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bf224df7-60f8-4803-be18-300d960fcc42","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","mode":"IntegratedOAuth","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67131949-3863-41f4-bed4-a5d7ca9baef5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67131949-3863-41f4-bed4-a5d7ca9baef5","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67131949-3863-41f4-bed4-a5d7ca9baef5","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"24b78c7f-c6ee-4077-b128-7eb729b060ea","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230"}
{"level":"info","ts":"2026-07-10T10:47:45.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67131949-3863-41f4-bed4-a5d7ca9baef5"}
{"level":"info","ts":"2026-07-10T10:47:45.333Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67131949-3863-41f4-bed4-a5d7ca9baef5","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:45.333Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2b7377aa-43d4-4519-b42f-2a6060cf4cae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.333Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2b7377aa-43d4-4519-b42f-2a6060cf4cae","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.333Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2b7377aa-43d4-4519-b42f-2a6060cf4cae","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.333Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2b7377aa-43d4-4519-b42f-2a6060cf4cae"}
{"level":"info","ts":"2026-07-10T10:47:45.333Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2b7377aa-43d4-4519-b42f-2a6060cf4cae","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:45.338Z","logger":"raycluster-resource","msg":"default","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.339Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-4h4dv","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","name":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:47:45.342Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","name":"fail-4h4dv","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:45.347Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"24b78c7f-c6ee-4077-b128-7eb729b060ea","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.347Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"f267109d-74e5-477d-8652-024856a9e180","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.351Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4bd621e-535c-45bd-bfd2-57d5691f1678","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.351Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4bd621e-535c-45bd-bfd2-57d5691f1678","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.351Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4bd621e-535c-45bd-bfd2-57d5691f1678","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.351Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4bd621e-535c-45bd-bfd2-57d5691f1678"}
{"level":"info","ts":"2026-07-10T10:47:45.351Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4bd621e-535c-45bd-bfd2-57d5691f1678","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:45.354Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","name":"fail-4h4dv","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"fail-4h4dv"}}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230"}
{"level":"error","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a04a4a75-b7c1-471d-9709-37b461322230","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6"}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6"}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:47:45.355Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","name":"fail-4h4dv","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:45.359Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"f267109d-74e5-477d-8652-024856a9e180","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:47:45.425Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.425Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.425Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6"}
{"level":"error","ts":"2026-07-10T10:47:45.425Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"675c35c9-9203-4675-9c0d-3eedf2b8a5e6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.425Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5"}
{"level":"info","ts":"2026-07-10T10:47:45.425Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5"}
{"level":"info","ts":"2026-07-10T10:47:45.425Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"18d48095-cf90-4485-a00b-3d9bbafe816f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"18d48095-cf90-4485-a00b-3d9bbafe816f","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"18d48095-cf90-4485-a00b-3d9bbafe816f","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","name":"fail-4h4dv","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"18d48095-cf90-4485-a00b-3d9bbafe816f"}
{"level":"info","ts":"2026-07-10T10:47:45.426Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"18d48095-cf90-4485-a00b-3d9bbafe816f","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:45.429Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"f267109d-74e5-477d-8652-024856a9e180","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:47:45.429Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"f267109d-74e5-477d-8652-024856a9e180","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.429Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b3cea918-85ad-4c3b-8da6-cf4392ed6ed9","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.431Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","name":"fail-4h4dv-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:47:45.433Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","name":"fail-4h4dv","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"fail-4h4dv"}}
{"level":"info","ts":"2026-07-10T10:47:45.433Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5"}
{"level":"error","ts":"2026-07-10T10:47:45.433Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"deed02a8-e1bc-46b7-9847-2dd0904eeef5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8"}
{"level":"error","ts":"2026-07-10T10:47:45.436Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5ab36045-ea4b-40cb-8d8a-9dac40b8bab8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.439Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg","operation":"created"}
{"level":"info","ts":"2026-07-10T10:47:45.445Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b3cea918-85ad-4c3b-8da6-cf4392ed6ed9","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:47:45.446Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.452Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","name":"test-ns-z4qpg-fail-4h4dv","operation":"created"}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b"}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b"}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","name":"kube-rbac-proxy-config-fail-4h4dv","operation":"created"}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b"}
{"level":"error","ts":"2026-07-10T10:47:45.525Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"78cc7fd4-c113-4657-91b4-9c4243dda61b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.529Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b3cea918-85ad-4c3b-8da6-cf4392ed6ed9","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:47:45.529Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b3cea918-85ad-4c3b-8da6-cf4392ed6ed9","cluster":"fail-4h4dv"}
{"level":"error","ts":"2026-07-10T10:47:45.532Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.532Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404"}
{"level":"error","ts":"2026-07-10T10:47:45.532Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e4fd384c-a2e0-4812-b276-6b2bfdd11404","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.532Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"871f5555-5168-4498-88d8-c478cebdf071","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.532Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"871f5555-5168-4498-88d8-c478cebdf071","mode":"IntegratedOAuth","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.532Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"871f5555-5168-4498-88d8-c478cebdf071","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.534Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","rayCluster":"fail-4h4dv","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:47:45.534Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-4h4dv\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.534Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218"}
{"level":"error","ts":"2026-07-10T10:47:45.534Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"94ff27f7-687d-4a81-96c3-5f7f261b5218","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-4h4dv\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.534Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"8a92a9f6-91d6-452e-9c9e-1786a27d2630","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.534Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"8a92a9f6-91d6-452e-9c9e-1786a27d2630"}
{"level":"info","ts":"2026-07-10T10:47:45.539Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"aac54588-8f60-4bae-836b-249a5360044e","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.540Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"aac54588-8f60-4bae-836b-249a5360044e"}
{"level":"info","ts":"2026-07-10T10:47:45.543Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"871f5555-5168-4498-88d8-c478cebdf071","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b51d90bf-15e5-4393-8570-e7eb9650d775","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b51d90bf-15e5-4393-8570-e7eb9650d775","mode":"IntegratedOAuth","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b51d90bf-15e5-4393-8570-e7eb9650d775","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c8680d1c-bf21-4f7e-867c-993db8c4f60b","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"df8bd521-257d-4efe-90fc-5ccd5329ea3f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"dd9ad756-943a-4079-b0a6-bc7c82a1ad9d","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"dd9ad756-943a-4079-b0a6-bc7c82a1ad9d"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"df8bd521-257d-4efe-90fc-5ccd5329ea3f","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"df8bd521-257d-4efe-90fc-5ccd5329ea3f","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"df8bd521-257d-4efe-90fc-5ccd5329ea3f"}
{"level":"info","ts":"2026-07-10T10:47:45.544Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"df8bd521-257d-4efe-90fc-5ccd5329ea3f","RayCluster":"fail-4h4dv","State":""}
{"level":"error","ts":"2026-07-10T10:47:45.553Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b51d90bf-15e5-4393-8570-e7eb9650d775","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.553Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b51d90bf-15e5-4393-8570-e7eb9650d775"}
{"level":"error","ts":"2026-07-10T10:47:45.553Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"b51d90bf-15e5-4393-8570-e7eb9650d775","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.558Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"8fa25ab0-72f7-43d5-a6ca-ffa7e43b204e","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:45.558Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"8fa25ab0-72f7-43d5-a6ca-ffa7e43b204e","mode":"IntegratedOAuth","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.558Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"8fa25ab0-72f7-43d5-a6ca-ffa7e43b204e","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.558Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c8680d1c-bf21-4f7e-867c-993db8c4f60b","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c8680d1c-bf21-4f7e-867c-993db8c4f60b","name":"fail-4h4dv-workers"}
{"level":"error","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c8680d1c-bf21-4f7e-867c-993db8c4f60b","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112"}
{"level":"error","ts":"2026-07-10T10:47:45.625Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2d812b6f-441a-4f47-b874-82c498a1e112","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.625Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"8fa25ab0-72f7-43d5-a6ca-ffa7e43b204e","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55"}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55"}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55"}
{"level":"error","ts":"2026-07-10T10:47:45.786Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"cbf9ded7-3cf6-43cd-ad4e-8fcb8dfdde55","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:46.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab"}
{"level":"info","ts":"2026-07-10T10:47:46.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab"}
{"level":"info","ts":"2026-07-10T10:47:46.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:46.106Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:46.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:46.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:46.107Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:46.107Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:46.107Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:46.107Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab"}
{"level":"error","ts":"2026-07-10T10:47:46.107Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77b1e136-890e-447f-ba62-953856a779ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a"}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a"}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","error":"head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"error","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found"}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-4h4dv","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a"}
{"level":"error","ts":"2026-07-10T10:47:46.748Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"bbd94f3e-37ea-4059-a602-13a16d8ad15a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-4h4dv not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:48.028Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce"}
{"level":"info","ts":"2026-07-10T10:47:48.028Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce"}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce"}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce"}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","mode":"IntegratedOAuth","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","cluster":"fail-4h4dv","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","cluster":"fail-4h4dv","authType":"IntegratedOAuth","serviceAccount":"fail-4h4dv-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-4h4dv","ray.io/group":"headgroup","ray.io/identifier":"fail-4h4dv-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:47:48.029Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","enabled":true}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","name":"fail-4h4dv-head-vxmcp"}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce"}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","nodeType":"worker","rayStartParams":{"address":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","rayStartCmd":"ray start  --address=fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:47:48.047Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","enabled":true}
{"level":"info","ts":"2026-07-10T10:47:48.063Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","name":"fail-4h4dv-small-group-worker-546nz"}
{"level":"info","ts":"2026-07-10T10:47:48.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.063Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-4h4dv-head-svc"},"newHead":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"}}
{"level":"info","ts":"2026-07-10T10:47:48.063Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5f6976a6-6de6-4e48-8a6e-f50b491fc0ce","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb"}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb"}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"41e6d05e-aaf6-4f7e-9545-c421178d6fcb","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6697cf6d-c722-460c-93e0-e48b4a2fe10e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:48.076Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e2860491-bcc2-41dc-a6e4-297ec9da4c6c","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"41e6d05e-aaf6-4f7e-9545-c421178d6fcb","rayCluster":"fail-4h4dv","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-4h4dv-head-svc"},"newHead":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"}}
{"level":"info","ts":"2026-07-10T10:47:48.077Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","name":"fail-4h4dv","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"fail-4h4dv"}}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb"}
{"level":"error","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ed79a8a6-84c9-4af2-9746-cadaa6a890fb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-4h4dv\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:48.086Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"43bef8d4-2038-415d-818f-ed52237c3716","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:48.125Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6697cf6d-c722-460c-93e0-e48b4a2fe10e","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.125Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6697cf6d-c722-460c-93e0-e48b4a2fe10e","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:48.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b"}
{"level":"info","ts":"2026-07-10T10:47:48.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b"}
{"level":"info","ts":"2026-07-10T10:47:48.125Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e2860491-bcc2-41dc-a6e4-297ec9da4c6c","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:47:48.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:47:48.127Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"f435abac-f9fa-4f23-b8c1-942dc15dac1b","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"56af5dcd-18e1-4893-83c5-d6b788e88a9b","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6697cf6d-c722-460c-93e0-e48b4a2fe10e"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6697cf6d-c722-460c-93e0-e48b4a2fe10e","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"56af5dcd-18e1-4893-83c5-d6b788e88a9b","rayCluster":"fail-4h4dv","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a5f8ce9c-7a45-45d5-b146-adfb6251dda6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a5f8ce9c-7a45-45d5-b146-adfb6251dda6","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a5f8ce9c-7a45-45d5-b146-adfb6251dda6","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.226Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a5f8ce9c-7a45-45d5-b146-adfb6251dda6"}
{"level":"info","ts":"2026-07-10T10:47:48.226Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a5f8ce9c-7a45-45d5-b146-adfb6251dda6","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:48.236Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e2860491-bcc2-41dc-a6e4-297ec9da4c6c","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:47:48.236Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e2860491-bcc2-41dc-a6e4-297ec9da4c6c","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.236Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"eeba98da-0918-4688-b0b3-65f944f53ab9","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-z4qpg"},"reconcileID":"c214603a-f978-4aa9-b739-68c606ed84b7"}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:48.325Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"16a8e83c-a458-43f1-8fbd-ff220315fc50","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:48.330Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"eeba98da-0918-4688-b0b3-65f944f53ab9","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:47:48.333Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"324dae7f-c322-4ab8-9786-2e7fb035bfd8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:48.333Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"324dae7f-c322-4ab8-9786-2e7fb035bfd8","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.333Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"324dae7f-c322-4ab8-9786-2e7fb035bfd8","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:48.333Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"324dae7f-c322-4ab8-9786-2e7fb035bfd8"}
{"level":"info","ts":"2026-07-10T10:47:48.333Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"324dae7f-c322-4ab8-9786-2e7fb035bfd8","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:48.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"eeba98da-0918-4688-b0b3-65f944f53ab9","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:47:48.341Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"eeba98da-0918-4688-b0b3-65f944f53ab9","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.501Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416"}
{"level":"info","ts":"2026-07-10T10:47:48.501Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416"}
{"level":"info","ts":"2026-07-10T10:47:48.501Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:48.502Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"ecc7d47f-fe4a-42b6-82d3-8c55fcce2416","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:48.518Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64c0b8f8-a77d-4c84-883c-fade2241920d","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:48.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d"}
{"level":"info","ts":"2026-07-10T10:47:48.677Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d"}
{"level":"info","ts":"2026-07-10T10:47:48.677Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:48.678Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a5fdfb72-da86-4575-90fa-db6e113a917d","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"newHead":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"}}
{"level":"info","ts":"2026-07-10T10:47:48.961Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"90d7ecc6-7d83-497a-b4b3-ff7548133c93","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"85ce7be8-2996-4e1e-8e18-2a125003f778","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bea184be-bb76-449d-829d-d34ed822442d","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"85ce7be8-2996-4e1e-8e18-2a125003f778","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"85ce7be8-2996-4e1e-8e18-2a125003f778","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"02be9293-6457-48c6-8be0-bacde2b829f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"02be9293-6457-48c6-8be0-bacde2b829f5","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"02be9293-6457-48c6-8be0-bacde2b829f5","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"02be9293-6457-48c6-8be0-bacde2b829f5"}
{"level":"info","ts":"2026-07-10T10:47:48.973Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"02be9293-6457-48c6-8be0-bacde2b829f5","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:48.985Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bea184be-bb76-449d-829d-d34ed822442d","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:47:48.986Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"85ce7be8-2996-4e1e-8e18-2a125003f778","certificate":"ray-head-cert-fail-4h4dv","dnsNames":["fail-4h4dv-head-svc","localhost","fail-4h4dv-head-svc.test-ns-z4qpg.svc","fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.31","10.133.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:47:48.998Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bea184be-bb76-449d-829d-d34ed822442d","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:47:48.998Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bea184be-bb76-449d-829d-d34ed822442d","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:48.998Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"85ce7be8-2996-4e1e-8e18-2a125003f778","certificate":"ray-worker-cert-fail-4h4dv","dnsNames":["fail-4h4dv-worker-svc","localhost","fail-4h4dv-small-group","fail-4h4dv-worker-svc.test-ns-z4qpg.svc","fail-4h4dv-worker-svc.test-ns-z4qpg.svc.cluster.local","*.fail-4h4dv-worker-svc.test-ns-z4qpg.svc","*.fail-4h4dv-worker-svc.test-ns-z4qpg.svc.cluster.local","*-worker-*.test-ns-z4qpg.svc","*-worker-*.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.31","10.133.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:47:48.998Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"85ce7be8-2996-4e1e-8e18-2a125003f778","rayCluster":"fail-4h4dv","podIPs":["10.132.0.31","10.133.0.42"]}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:50.226Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"77bd824b-0ed7-4dcc-acd2-2c56089fcde9","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:51.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"05576533-c7f9-4254-aa75-03b84831f019","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:51.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"05576533-c7f9-4254-aa75-03b84831f019","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:51.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"05576533-c7f9-4254-aa75-03b84831f019","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:51.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"05576533-c7f9-4254-aa75-03b84831f019"}
{"level":"info","ts":"2026-07-10T10:47:51.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"05576533-c7f9-4254-aa75-03b84831f019","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","podName":"fail-4h4dv-head-vxmcp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:53.982Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"2882acca-7342-45f1-b738-c012183ea6d6","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:54.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bfa11e0c-1c49-4ade-8d6d-cbfabbb6e90f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:54.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bfa11e0c-1c49-4ade-8d6d-cbfabbb6e90f","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:54.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bfa11e0c-1c49-4ade-8d6d-cbfabbb6e90f","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:54.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bfa11e0c-1c49-4ade-8d6d-cbfabbb6e90f"}
{"level":"info","ts":"2026-07-10T10:47:54.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bfa11e0c-1c49-4ade-8d6d-cbfabbb6e90f","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81"}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81"}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","podName":"fail-4h4dv-head-vxmcp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","podName":"fail-4h4dv-head-vxmcp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:47:54.992Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"634d0ca4-90af-4c39-9e43-a804566dac81","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bc8c050a-890d-43c3-8f1e-dcff26a0bfc0","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cf71fce-bd36-4e9c-acce-1b748e0221e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cf71fce-bd36-4e9c-acce-1b748e0221e3","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cf71fce-bd36-4e9c-acce-1b748e0221e3","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7d6e00bc-303d-435e-afe3-03f20d92eace","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bc8c050a-890d-43c3-8f1e-dcff26a0bfc0","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bc8c050a-890d-43c3-8f1e-dcff26a0bfc0","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"bc8c050a-890d-43c3-8f1e-dcff26a0bfc0","rayCluster":"fail-4h4dv","podIPs":["10.133.0.42","10.132.0.31"]}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cf71fce-bd36-4e9c-acce-1b748e0221e3"}
{"level":"info","ts":"2026-07-10T10:47:55.006Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cf71fce-bd36-4e9c-acce-1b748e0221e3","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:55.026Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7d6e00bc-303d-435e-afe3-03f20d92eace","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:47:55.036Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7d6e00bc-303d-435e-afe3-03f20d92eace","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:47:55.036Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7d6e00bc-303d-435e-afe3-03f20d92eace","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd"}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd"}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","podName":"fail-4h4dv-head-vxmcp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","podName":"fail-4h4dv-head-vxmcp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:57.007Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"445dac37-4f5e-4126-8852-4122c21386dd","seconds":300}
{"level":"info","ts":"2026-07-10T10:47:57.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a8640648-01c7-4070-b58b-7e3ad15c534c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:47:57.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a8640648-01c7-4070-b58b-7e3ad15c534c","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:47:57.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a8640648-01c7-4070-b58b-7e3ad15c534c","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:47:57.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a8640648-01c7-4070-b58b-7e3ad15c534c"}
{"level":"info","ts":"2026-07-10T10:47:57.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a8640648-01c7-4070-b58b-7e3ad15c534c","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:47:59.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a"}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a"}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:47:59.926Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"54e516cc-f39a-4e84-8dad-e46c067dd29a","seconds":300}
{"level":"info","ts":"2026-07-10T10:48:00.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4f5c815-a359-4283-83da-686dde8abc88","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:00.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4f5c815-a359-4283-83da-686dde8abc88","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:00.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4f5c815-a359-4283-83da-686dde8abc88","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:00.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4f5c815-a359-4283-83da-686dde8abc88"}
{"level":"info","ts":"2026-07-10T10:48:00.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d4f5c815-a359-4283-83da-686dde8abc88","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:03.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c22bc18-5668-4746-bf84-99374c2d30c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:03.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c22bc18-5668-4746-bf84-99374c2d30c1","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:03.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c22bc18-5668-4746-bf84-99374c2d30c1","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:03.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c22bc18-5668-4746-bf84-99374c2d30c1"}
{"level":"info","ts":"2026-07-10T10:48:03.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c22bc18-5668-4746-bf84-99374c2d30c1","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:48:05.044Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"64b18124-1de7-4d31-bdf6-627c7dce9ae2","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:47:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:48:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e3872ff6-664f-485a-a0ac-2bde19ce06c0","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3c2065c0-639d-4c4d-86f6-e96bd45a5e69","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d97ddba3-bc42-4c4f-a7e1-87338e46a228","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d97ddba3-bc42-4c4f-a7e1-87338e46a228","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d97ddba3-bc42-4c4f-a7e1-87338e46a228","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3c2065c0-639d-4c4d-86f6-e96bd45a5e69","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3c2065c0-639d-4c4d-86f6-e96bd45a5e69","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d97ddba3-bc42-4c4f-a7e1-87338e46a228"}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d97ddba3-bc42-4c4f-a7e1-87338e46a228","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:05.058Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3c2065c0-639d-4c4d-86f6-e96bd45a5e69","rayCluster":"fail-4h4dv","podIPs":["10.132.0.31","10.133.0.42"]}
{"level":"info","ts":"2026-07-10T10:48:05.071Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e3872ff6-664f-485a-a0ac-2bde19ce06c0","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:48:05.082Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e3872ff6-664f-485a-a0ac-2bde19ce06c0","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:48:05.082Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"e3872ff6-664f-485a-a0ac-2bde19ce06c0","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:06.339Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"53252b69-023e-4406-aac3-f0389901bfdc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:06.339Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"53252b69-023e-4406-aac3-f0389901bfdc","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:06.339Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"53252b69-023e-4406-aac3-f0389901bfdc","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:06.339Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"53252b69-023e-4406-aac3-f0389901bfdc"}
{"level":"info","ts":"2026-07-10T10:48:06.339Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"53252b69-023e-4406-aac3-f0389901bfdc","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:07.057Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d"}
{"level":"info","ts":"2026-07-10T10:48:07.057Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d"}
{"level":"info","ts":"2026-07-10T10:48:07.057Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:48:07.058Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"8f53e6c7-5480-4ddf-83d8-6e7a5cc1586d","seconds":300}
{"level":"info","ts":"2026-07-10T10:48:09.340Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"69af5c04-9dd0-47e5-87cc-a067803c7c5c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:09.340Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"69af5c04-9dd0-47e5-87cc-a067803c7c5c","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:09.340Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"69af5c04-9dd0-47e5-87cc-a067803c7c5c","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:09.340Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"69af5c04-9dd0-47e5-87cc-a067803c7c5c"}
{"level":"info","ts":"2026-07-10T10:48:09.340Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"69af5c04-9dd0-47e5-87cc-a067803c7c5c","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:10.428Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8e43f0c-56d0-4d89-b4d7-6845c6c6d4c9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:48:10.428Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8e43f0c-56d0-4d89-b4d7-6845c6c6d4c9","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:48:10.428Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8e43f0c-56d0-4d89-b4d7-6845c6c6d4c9","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:48:10.428Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8e43f0c-56d0-4d89-b4d7-6845c6c6d4c9","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:48:12.341Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7db2a9e3-3e77-477f-991e-c9f55240f5b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:12.341Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7db2a9e3-3e77-477f-991e-c9f55240f5b9","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:12.341Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7db2a9e3-3e77-477f-991e-c9f55240f5b9","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:12.341Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7db2a9e3-3e77-477f-991e-c9f55240f5b9"}
{"level":"info","ts":"2026-07-10T10:48:12.341Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7db2a9e3-3e77-477f-991e-c9f55240f5b9","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:15.342Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"918efd02-dedd-45f2-834b-f8f354624ba5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:15.342Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"918efd02-dedd-45f2-834b-f8f354624ba5","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:15.342Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"918efd02-dedd-45f2-834b-f8f354624ba5","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:15.342Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"918efd02-dedd-45f2-834b-f8f354624ba5"}
{"level":"info","ts":"2026-07-10T10:48:15.342Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"918efd02-dedd-45f2-834b-f8f354624ba5","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:15.540Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c5bf898c-219c-40f8-9a49-9dbc4297e112","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:15.540Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c5bf898c-219c-40f8-9a49-9dbc4297e112","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:48:15.540Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c5bf898c-219c-40f8-9a49-9dbc4297e112","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:48:15.540Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c5bf898c-219c-40f8-9a49-9dbc4297e112","rayCluster":"fail-4h4dv","podIPs":["10.132.0.31","10.133.0.42"]}
{"level":"info","ts":"2026-07-10T10:48:18.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8c172328-77f9-40a3-bbe6-3f0fe5ea9c22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:18.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8c172328-77f9-40a3-bbe6-3f0fe5ea9c22","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:18.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8c172328-77f9-40a3-bbe6-3f0fe5ea9c22","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:18.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8c172328-77f9-40a3-bbe6-3f0fe5ea9c22"}
{"level":"info","ts":"2026-07-10T10:48:18.343Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8c172328-77f9-40a3-bbe6-3f0fe5ea9c22","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:21.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a6d22082-fe6e-4aa9-bd7d-efb5ca0ee0e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:21.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a6d22082-fe6e-4aa9-bd7d-efb5ca0ee0e6","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:21.344Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a6d22082-fe6e-4aa9-bd7d-efb5ca0ee0e6","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:21.344Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a6d22082-fe6e-4aa9-bd7d-efb5ca0ee0e6"}
{"level":"info","ts":"2026-07-10T10:48:21.344Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"a6d22082-fe6e-4aa9-bd7d-efb5ca0ee0e6","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:24.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a56d4a-597d-4395-a341-b59a73eb3d45","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:24.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a56d4a-597d-4395-a341-b59a73eb3d45","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:24.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a56d4a-597d-4395-a341-b59a73eb3d45","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:24.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a56d4a-597d-4395-a341-b59a73eb3d45"}
{"level":"info","ts":"2026-07-10T10:48:24.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a56d4a-597d-4395-a341-b59a73eb3d45","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:27.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cc14963-032f-4482-a986-048b132913ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:27.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cc14963-032f-4482-a986-048b132913ce","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:27.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cc14963-032f-4482-a986-048b132913ce","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:27.346Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cc14963-032f-4482-a986-048b132913ce"}
{"level":"info","ts":"2026-07-10T10:48:27.346Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8cc14963-032f-4482-a986-048b132913ce","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:30.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c4b6b0ce-c91f-48c6-a73d-97bd814ccb85","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:30.347Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c4b6b0ce-c91f-48c6-a73d-97bd814ccb85","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:30.347Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c4b6b0ce-c91f-48c6-a73d-97bd814ccb85","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:30.347Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c4b6b0ce-c91f-48c6-a73d-97bd814ccb85"}
{"level":"info","ts":"2026-07-10T10:48:30.347Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c4b6b0ce-c91f-48c6-a73d-97bd814ccb85","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:33.348Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"0abef66f-7458-4fbe-bd2b-1a98da975fe3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:33.348Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"0abef66f-7458-4fbe-bd2b-1a98da975fe3","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:33.348Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"0abef66f-7458-4fbe-bd2b-1a98da975fe3","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:33.348Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"0abef66f-7458-4fbe-bd2b-1a98da975fe3"}
{"level":"info","ts":"2026-07-10T10:48:33.348Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"0abef66f-7458-4fbe-bd2b-1a98da975fe3","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:36.349Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"90270865-b2a4-43c4-acab-5787495e07e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:36.349Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"90270865-b2a4-43c4-acab-5787495e07e4","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:36.349Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"90270865-b2a4-43c4-acab-5787495e07e4","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:36.349Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"90270865-b2a4-43c4-acab-5787495e07e4"}
{"level":"info","ts":"2026-07-10T10:48:36.349Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"90270865-b2a4-43c4-acab-5787495e07e4","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:39.350Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"beb3a955-98e8-4c35-9e07-a70f306f540f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:39.350Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"beb3a955-98e8-4c35-9e07-a70f306f540f","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:39.350Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"beb3a955-98e8-4c35-9e07-a70f306f540f","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:39.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"beb3a955-98e8-4c35-9e07-a70f306f540f"}
{"level":"info","ts":"2026-07-10T10:48:39.350Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"beb3a955-98e8-4c35-9e07-a70f306f540f","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:42.351Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c57d7d0-0c13-415a-9773-148e5a3b5296","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:42.351Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c57d7d0-0c13-415a-9773-148e5a3b5296","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:42.351Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c57d7d0-0c13-415a-9773-148e5a3b5296","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:42.351Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c57d7d0-0c13-415a-9773-148e5a3b5296"}
{"level":"info","ts":"2026-07-10T10:48:42.351Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6c57d7d0-0c13-415a-9773-148e5a3b5296","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:45.352Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"04625682-de86-439f-86c7-91b8bfae4f0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:45.352Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"04625682-de86-439f-86c7-91b8bfae4f0b","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:45.352Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"04625682-de86-439f-86c7-91b8bfae4f0b","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:45.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"04625682-de86-439f-86c7-91b8bfae4f0b"}
{"level":"info","ts":"2026-07-10T10:48:45.352Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"04625682-de86-439f-86c7-91b8bfae4f0b","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:48.353Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6383935d-1ec8-4253-b393-da4d18bbe8f8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:48.353Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6383935d-1ec8-4253-b393-da4d18bbe8f8","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:48.353Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6383935d-1ec8-4253-b393-da4d18bbe8f8","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:48.353Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6383935d-1ec8-4253-b393-da4d18bbe8f8"}
{"level":"info","ts":"2026-07-10T10:48:48.353Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6383935d-1ec8-4253-b393-da4d18bbe8f8","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:51.354Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"20bce551-fbb0-402c-b0a8-3b2ec96d1d00","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:51.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"20bce551-fbb0-402c-b0a8-3b2ec96d1d00","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:51.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"20bce551-fbb0-402c-b0a8-3b2ec96d1d00","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:51.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"20bce551-fbb0-402c-b0a8-3b2ec96d1d00"}
{"level":"info","ts":"2026-07-10T10:48:51.355Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"20bce551-fbb0-402c-b0a8-3b2ec96d1d00","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:53.963Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944"}
{"level":"info","ts":"2026-07-10T10:48:53.963Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944"}
{"level":"info","ts":"2026-07-10T10:48:53.963Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:48:53.964Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7e0c5cf5-2260-4cac-8e73-fb87b3afd944","seconds":300}
{"level":"info","ts":"2026-07-10T10:48:54.356Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b0c2059b-156c-4094-bb11-7a4be4b34af7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:54.356Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b0c2059b-156c-4094-bb11-7a4be4b34af7","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:54.356Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b0c2059b-156c-4094-bb11-7a4be4b34af7","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:54.356Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b0c2059b-156c-4094-bb11-7a4be4b34af7"}
{"level":"info","ts":"2026-07-10T10:48:54.356Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b0c2059b-156c-4094-bb11-7a4be4b34af7","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:48:54.969Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a4d01ada-67af-41da-bc38-e291ec61c701","seconds":300}
{"level":"info","ts":"2026-07-10T10:48:57.356Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ceec370b-2008-4538-92a0-416788bd16d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:48:57.356Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ceec370b-2008-4538-92a0-416788bd16d3","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:48:57.356Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ceec370b-2008-4538-92a0-416788bd16d3","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:48:57.357Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ceec370b-2008-4538-92a0-416788bd16d3"}
{"level":"info","ts":"2026-07-10T10:48:57.357Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ceec370b-2008-4538-92a0-416788bd16d3","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:00.357Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d41b5585-6efa-4756-831e-ddf93bd923ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:00.357Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d41b5585-6efa-4756-831e-ddf93bd923ee","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:00.357Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d41b5585-6efa-4756-831e-ddf93bd923ee","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:00.358Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d41b5585-6efa-4756-831e-ddf93bd923ee"}
{"level":"info","ts":"2026-07-10T10:49:00.358Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"d41b5585-6efa-4756-831e-ddf93bd923ee","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:03.358Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ccfb66d3-0c36-4168-a1aa-68f7118af6d6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:03.358Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ccfb66d3-0c36-4168-a1aa-68f7118af6d6","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:03.358Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ccfb66d3-0c36-4168-a1aa-68f7118af6d6","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:03.358Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ccfb66d3-0c36-4168-a1aa-68f7118af6d6"}
{"level":"info","ts":"2026-07-10T10:49:03.358Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"ccfb66d3-0c36-4168-a1aa-68f7118af6d6","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:04.011Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"75ddd3b9-b376-4262-9005-379fbf0e38c1","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc"}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc"}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:49:05.018Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"07d14931-5876-42af-ae7d-afdfe07812fc","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:48:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7a1b181d-7ba1-4a89-96d8-e8c5f06501f0","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"6a43c457-15e1-4b30-9635-95fb18aaf9b1","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7a1b181d-7ba1-4a89-96d8-e8c5f06501f0","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7a1b181d-7ba1-4a89-96d8-e8c5f06501f0","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"970f1f72-3da5-4c15-90b5-f22de7312554","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"970f1f72-3da5-4c15-90b5-f22de7312554","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"970f1f72-3da5-4c15-90b5-f22de7312554","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"7a1b181d-7ba1-4a89-96d8-e8c5f06501f0","rayCluster":"fail-4h4dv","podIPs":["10.132.0.31","10.133.0.42"]}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"970f1f72-3da5-4c15-90b5-f22de7312554"}
{"level":"info","ts":"2026-07-10T10:49:05.033Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"970f1f72-3da5-4c15-90b5-f22de7312554","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:05.049Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"6a43c457-15e1-4b30-9635-95fb18aaf9b1","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:49:05.061Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"6a43c457-15e1-4b30-9635-95fb18aaf9b1","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:49:05.061Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"6a43c457-15e1-4b30-9635-95fb18aaf9b1","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:06.359Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7371d7ed-4f8e-4868-ba23-2027cfc60b33","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:06.359Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7371d7ed-4f8e-4868-ba23-2027cfc60b33","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:06.359Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7371d7ed-4f8e-4868-ba23-2027cfc60b33","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:06.359Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7371d7ed-4f8e-4868-ba23-2027cfc60b33"}
{"level":"info","ts":"2026-07-10T10:49:06.359Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7371d7ed-4f8e-4868-ba23-2027cfc60b33","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3"}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3"}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","podName":"fail-4h4dv-small-group-worker-546nz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:47:48Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:07.034Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"7bf05adf-7e1e-4bfc-a42e-6f69b540bce3","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:09.360Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"f3e162cd-94d6-4d9c-8c11-4ffd11bde928","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:09.360Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"f3e162cd-94d6-4d9c-8c11-4ffd11bde928","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:09.360Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"f3e162cd-94d6-4d9c-8c11-4ffd11bde928","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:09.360Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"f3e162cd-94d6-4d9c-8c11-4ffd11bde928"}
{"level":"info","ts":"2026-07-10T10:49:09.360Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"f3e162cd-94d6-4d9c-8c11-4ffd11bde928","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:10.429Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5d531b27-7bba-4992-99e5-382a68fc4650","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:49:10.429Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5d531b27-7bba-4992-99e5-382a68fc4650","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:49:10.429Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5d531b27-7bba-4992-99e5-382a68fc4650","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:49:10.429Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5d531b27-7bba-4992-99e5-382a68fc4650","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:49:12.361Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e9cc1f88-ab1c-4ba3-abd7-03a9d48cc938","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:12.361Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e9cc1f88-ab1c-4ba3-abd7-03a9d48cc938","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:12.361Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e9cc1f88-ab1c-4ba3-abd7-03a9d48cc938","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:12.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e9cc1f88-ab1c-4ba3-abd7-03a9d48cc938"}
{"level":"info","ts":"2026-07-10T10:49:12.361Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e9cc1f88-ab1c-4ba3-abd7-03a9d48cc938","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:15.361Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"11486987-63c1-47ae-b4f4-17f3d6f1ba3c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:15.361Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"11486987-63c1-47ae-b4f4-17f3d6f1ba3c","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:15.361Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"11486987-63c1-47ae-b4f4-17f3d6f1ba3c","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:15.362Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"11486987-63c1-47ae-b4f4-17f3d6f1ba3c"}
{"level":"info","ts":"2026-07-10T10:49:15.362Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"11486987-63c1-47ae-b4f4-17f3d6f1ba3c","RayCluster":"fail-4h4dv","State":""}
{"level":"info","ts":"2026-07-10T10:49:15.541Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"1cef1c22-ff6c-47cc-a61f-6b81ad387d3a","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:15.541Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"1cef1c22-ff6c-47cc-a61f-6b81ad387d3a","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:49:15.541Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"1cef1c22-ff6c-47cc-a61f-6b81ad387d3a","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:49:15.541Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"1cef1c22-ff6c-47cc-a61f-6b81ad387d3a","rayCluster":"fail-4h4dv","podIPs":["10.132.0.31","10.133.0.42"]}
{"level":"info","ts":"2026-07-10T10:49:16.045Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e"}
{"level":"info","ts":"2026-07-10T10:49:16.045Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e"}
{"level":"info","ts":"2026-07-10T10:49:16.045Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:16.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:49:16.046Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"a85f08ca-fb6b-4c8a-b78e-f07062fbe67e","name":"fail-4h4dv","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"5c99836a-2e34-4d27-9369-dd76314c2406","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"9a8f37c9-8d16-464c-bc41-1d6de1896194","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"9a8f37c9-8d16-464c-bc41-1d6de1896194","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"9a8f37c9-8d16-464c-bc41-1d6de1896194","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","head service name":"fail-4h4dv-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","head service URL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123"}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","command":["if","!","ray","job","status","--address","http://fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","fail-q4crc",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","--no-wait","--submission-id","fail-q4crc","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","--follow","fail-q4crc"]}
{"level":"info","ts":"2026-07-10T10:49:16.060Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"9a8f37c9-8d16-464c-bc41-1d6de1896194","rayCluster":"fail-4h4dv","podIPs":["10.133.0.42","10.132.0.31"]}
{"level":"info","ts":"2026-07-10T10:49:16.070Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-07-10T10:49:16.070Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","SubmissionMode":"K8sJobMode","RayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.070Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:49:16.070Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"689cde33-3e68-45fa-babe-c94facd44123","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:49:16.076Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"5c99836a-2e34-4d27-9369-dd76314c2406","name":"fail-4h4dv-head"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","head service name":"fail-4h4dv-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","head service URL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","SubmissionMode":"K8sJobMode","RayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:49:16.082Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:49:16.091Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"5c99836a-2e34-4d27-9369-dd76314c2406","name":"fail-4h4dv-workers"}
{"level":"info","ts":"2026-07-10T10:49:16.091Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"5c99836a-2e34-4d27-9369-dd76314c2406","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.096Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:49:16.096Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b"}
{"level":"error","ts":"2026-07-10T10:49:16.096Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"2ea26f84-2530-4611-889e-50394fe1eb3b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.096Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c34e09af-ae36-414c-a02a-718be64503ea","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.096Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c34e09af-ae36-414c-a02a-718be64503ea","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.096Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c34e09af-ae36-414c-a02a-718be64503ea","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.096Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c34e09af-ae36-414c-a02a-718be64503ea"}
{"level":"error","ts":"2026-07-10T10:49:16.133Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c34e09af-ae36-414c-a02a-718be64503ea","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.133Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c34e09af-ae36-414c-a02a-718be64503ea"}
{"level":"error","ts":"2026-07-10T10:49:16.133Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c34e09af-ae36-414c-a02a-718be64503ea","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.133Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"93cc43ea-d469-4bca-8911-75b3d128f461","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.133Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"93cc43ea-d469-4bca-8911-75b3d128f461","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.133Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"93cc43ea-d469-4bca-8911-75b3d128f461","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.133Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"93cc43ea-d469-4bca-8911-75b3d128f461"}
{"level":"error","ts":"2026-07-10T10:49:16.138Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"93cc43ea-d469-4bca-8911-75b3d128f461","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.138Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"93cc43ea-d469-4bca-8911-75b3d128f461"}
{"level":"error","ts":"2026-07-10T10:49:16.138Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"93cc43ea-d469-4bca-8911-75b3d128f461","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.143Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c9c4929c-9b12-4393-83c3-4606c21edbd5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.143Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c9c4929c-9b12-4393-83c3-4606c21edbd5","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.143Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c9c4929c-9b12-4393-83c3-4606c21edbd5","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.144Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c9c4929c-9b12-4393-83c3-4606c21edbd5"}
{"level":"error","ts":"2026-07-10T10:49:16.148Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c9c4929c-9b12-4393-83c3-4606c21edbd5","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.148Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c9c4929c-9b12-4393-83c3-4606c21edbd5"}
{"level":"error","ts":"2026-07-10T10:49:16.148Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"c9c4929c-9b12-4393-83c3-4606c21edbd5","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.189Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"231267d0-0625-48e3-9e5d-4ba35a1bdd52","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.189Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"231267d0-0625-48e3-9e5d-4ba35a1bdd52","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.189Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"231267d0-0625-48e3-9e5d-4ba35a1bdd52","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.189Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"231267d0-0625-48e3-9e5d-4ba35a1bdd52"}
{"level":"error","ts":"2026-07-10T10:49:16.194Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"231267d0-0625-48e3-9e5d-4ba35a1bdd52","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.194Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"231267d0-0625-48e3-9e5d-4ba35a1bdd52"}
{"level":"error","ts":"2026-07-10T10:49:16.194Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"231267d0-0625-48e3-9e5d-4ba35a1bdd52","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.275Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"360555f3-9ebc-4c34-b914-5606f3512e5d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.275Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"360555f3-9ebc-4c34-b914-5606f3512e5d","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.275Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"360555f3-9ebc-4c34-b914-5606f3512e5d","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.275Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"360555f3-9ebc-4c34-b914-5606f3512e5d"}
{"level":"error","ts":"2026-07-10T10:49:16.281Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"360555f3-9ebc-4c34-b914-5606f3512e5d","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.281Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"360555f3-9ebc-4c34-b914-5606f3512e5d"}
{"level":"error","ts":"2026-07-10T10:49:16.281Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"360555f3-9ebc-4c34-b914-5606f3512e5d","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.441Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e871f3e6-5933-419b-97f1-10571f6b7a0b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.441Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e871f3e6-5933-419b-97f1-10571f6b7a0b","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.441Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e871f3e6-5933-419b-97f1-10571f6b7a0b","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.441Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e871f3e6-5933-419b-97f1-10571f6b7a0b"}
{"level":"error","ts":"2026-07-10T10:49:16.447Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e871f3e6-5933-419b-97f1-10571f6b7a0b","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.447Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e871f3e6-5933-419b-97f1-10571f6b7a0b"}
{"level":"error","ts":"2026-07-10T10:49:16.447Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e871f3e6-5933-419b-97f1-10571f6b7a0b","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6215ed7a-8c86-4c12-bb1e-f78ef9f2e94c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:16.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6215ed7a-8c86-4c12-bb1e-f78ef9f2e94c","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:16.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6215ed7a-8c86-4c12-bb1e-f78ef9f2e94c","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:16.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6215ed7a-8c86-4c12-bb1e-f78ef9f2e94c"}
{"level":"error","ts":"2026-07-10T10:49:16.773Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6215ed7a-8c86-4c12-bb1e-f78ef9f2e94c","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:16.773Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6215ed7a-8c86-4c12-bb1e-f78ef9f2e94c"}
{"level":"error","ts":"2026-07-10T10:49:16.773Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"6215ed7a-8c86-4c12-bb1e-f78ef9f2e94c","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:17.095Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3e84af74-af63-4e59-a931-3eab0338e19a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:17.095Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3e84af74-af63-4e59-a931-3eab0338e19a","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:17.095Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3e84af74-af63-4e59-a931-3eab0338e19a","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:17.095Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3e84af74-af63-4e59-a931-3eab0338e19a"}
{"level":"error","ts":"2026-07-10T10:49:17.101Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3e84af74-af63-4e59-a931-3eab0338e19a","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:17.101Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3e84af74-af63-4e59-a931-3eab0338e19a"}
{"level":"error","ts":"2026-07-10T10:49:17.101Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3e84af74-af63-4e59-a931-3eab0338e19a","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:17.414Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"9dc144d8-f7dc-43e4-9dc4-19d1412c9214","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:17.414Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"9dc144d8-f7dc-43e4-9dc4-19d1412c9214","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:17.414Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"9dc144d8-f7dc-43e4-9dc4-19d1412c9214","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:17.414Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"9dc144d8-f7dc-43e4-9dc4-19d1412c9214"}
{"level":"error","ts":"2026-07-10T10:49:17.419Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"9dc144d8-f7dc-43e4-9dc4-19d1412c9214","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:17.419Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"9dc144d8-f7dc-43e4-9dc4-19d1412c9214"}
{"level":"error","ts":"2026-07-10T10:49:17.419Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"9dc144d8-f7dc-43e4-9dc4-19d1412c9214","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:18.061Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b"}
{"level":"info","ts":"2026-07-10T10:49:18.061Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b"}
{"level":"info","ts":"2026-07-10T10:49:18.061Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","1 head service found":"fail-4h4dv-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:18.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","Found 1 head Pod":"fail-4h4dv-head-vxmcp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:18.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","head Pod":"fail-4h4dv-head-vxmcp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-4h4dv-head-vxmcp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:18.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:18.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","worker Pod":"fail-4h4dv-small-group-worker-546nz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-4h4dv-small-group-worker-546nz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:18.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:18.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:18.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:18.062Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:18.062Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"03bcbc58-bc58-402a-bb62-8ca1d0e24d9b","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:19.980Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67bf8299-90e4-4c92-8076-0b3b7e38f033","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:19.980Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67bf8299-90e4-4c92-8076-0b3b7e38f033","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:19.980Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67bf8299-90e4-4c92-8076-0b3b7e38f033","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:19.980Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67bf8299-90e4-4c92-8076-0b3b7e38f033"}
{"level":"error","ts":"2026-07-10T10:49:19.986Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67bf8299-90e4-4c92-8076-0b3b7e38f033","JobId":"fail-q4crc","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:19.986Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67bf8299-90e4-4c92-8076-0b3b7e38f033"}
{"level":"error","ts":"2026-07-10T10:49:19.986Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67bf8299-90e4-4c92-8076-0b3b7e38f033","error":"Job fail-q4crc does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:25.106Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e58bd6c3-d121-4a37-bf55-24028ced371d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:25.106Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e58bd6c3-d121-4a37-bf55-24028ced371d","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:25.106Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e58bd6c3-d121-4a37-bf55-24028ced371d","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:25.106Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e58bd6c3-d121-4a37-bf55-24028ced371d"}
{"level":"info","ts":"2026-07-10T10:49:25.112Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e58bd6c3-d121-4a37-bf55-24028ced371d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:25.112Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"e58bd6c3-d121-4a37-bf55-24028ced371d","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:49:25.127Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"edb37081-a1ce-4e5f-803e-ac0289724419","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:25.127Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"edb37081-a1ce-4e5f-803e-ac0289724419","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:25.127Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"edb37081-a1ce-4e5f-803e-ac0289724419","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:25.127Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"edb37081-a1ce-4e5f-803e-ac0289724419"}
{"level":"info","ts":"2026-07-10T10:49:25.130Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"edb37081-a1ce-4e5f-803e-ac0289724419","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:28.128Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a6e5a2-380b-41df-a8bf-216cf97ff2fa","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:28.128Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a6e5a2-380b-41df-a8bf-216cf97ff2fa","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:28.128Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a6e5a2-380b-41df-a8bf-216cf97ff2fa","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:28.128Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a6e5a2-380b-41df-a8bf-216cf97ff2fa"}
{"level":"info","ts":"2026-07-10T10:49:28.131Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a6e5a2-380b-41df-a8bf-216cf97ff2fa","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:28.131Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"91a6e5a2-380b-41df-a8bf-216cf97ff2fa","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:49:28.149Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"cf0bb78d-9cf6-4f3d-ab87-6c1ba8ff7edb","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:28.149Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"cf0bb78d-9cf6-4f3d-ab87-6c1ba8ff7edb","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:28.149Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"cf0bb78d-9cf6-4f3d-ab87-6c1ba8ff7edb","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:28.149Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"cf0bb78d-9cf6-4f3d-ab87-6c1ba8ff7edb"}
{"level":"info","ts":"2026-07-10T10:49:28.152Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"cf0bb78d-9cf6-4f3d-ab87-6c1ba8ff7edb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:31.148Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bbf55da4-920a-456a-a546-cfdbee1bcbf3","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:31.148Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bbf55da4-920a-456a-a546-cfdbee1bcbf3","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:31.148Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bbf55da4-920a-456a-a546-cfdbee1bcbf3","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:31.148Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bbf55da4-920a-456a-a546-cfdbee1bcbf3"}
{"level":"info","ts":"2026-07-10T10:49:31.151Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"bbf55da4-920a-456a-a546-cfdbee1bcbf3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:34.152Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7a7aaad1-6672-4e1e-af29-0e85a221484a","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:34.152Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7a7aaad1-6672-4e1e-af29-0e85a221484a","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:34.152Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7a7aaad1-6672-4e1e-af29-0e85a221484a","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:34.152Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7a7aaad1-6672-4e1e-af29-0e85a221484a"}
{"level":"info","ts":"2026-07-10T10:49:34.155Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"7a7aaad1-6672-4e1e-af29-0e85a221484a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:35.165Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8383b9e7-32b4-4719-a4cc-77d68e228733","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:35.165Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8383b9e7-32b4-4719-a4cc-77d68e228733","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:35.165Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8383b9e7-32b4-4719-a4cc-77d68e228733","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:35.165Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8383b9e7-32b4-4719-a4cc-77d68e228733"}
{"level":"info","ts":"2026-07-10T10:49:35.169Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"8383b9e7-32b4-4719-a4cc-77d68e228733","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:36.173Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b641093a-1daa-4f3f-b03b-719bb6b6b82b","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:36.173Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b641093a-1daa-4f3f-b03b-719bb6b6b82b","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:36.173Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b641093a-1daa-4f3f-b03b-719bb6b6b82b","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:36.173Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b641093a-1daa-4f3f-b03b-719bb6b6b82b"}
{"level":"info","ts":"2026-07-10T10:49:36.177Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b641093a-1daa-4f3f-b03b-719bb6b6b82b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:36.190Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"aaeaf06e-491a-4c39-880d-a4f304550e3f","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:36.191Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"aaeaf06e-491a-4c39-880d-a4f304550e3f","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:36.191Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"aaeaf06e-491a-4c39-880d-a4f304550e3f","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:36.191Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"aaeaf06e-491a-4c39-880d-a4f304550e3f"}
{"level":"info","ts":"2026-07-10T10:49:36.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"aaeaf06e-491a-4c39-880d-a4f304550e3f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:49:23Z","endTime":"2026-07-10T10:49:25Z"},"jobId":"fail-q4crc","rayClusterName":"fail-4h4dv","dashboardURL":"fail-4h4dv-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:49:23,911\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-q4crc: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:47:45Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:16Z","stateTransitionTimes":{"ready":"2026-07-10T10:49:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"fail-4h4dv-head-vxmcp","serviceName":"fail-4h4dv-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:48:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:49:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:47:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:47:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:49:36.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"aaeaf06e-491a-4c39-880d-a4f304550e3f","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-10T10:49:36.210Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b6338dc1-3d76-4672-a441-6f6610555cf3","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:36.210Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b6338dc1-3d76-4672-a441-6f6610555cf3","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:49:36 +0000 UTC","Now":"2026-07-10T10:49:36.210Z","ShutdownTime":"2026-07-10T10:49:36.000Z"}
{"level":"info","ts":"2026-07-10T10:49:36.977Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67acde4c-ea4c-4cf8-afc9-99501e96df78","DeletionTimestamp":"2026-07-10 10:49:36 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:49:36.977Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"67acde4c-ea4c-4cf8-afc9-99501e96df78","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:49:36.994Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"3ee7eb38-4ae2-44e5-a28f-56b5121ea35b"}
{"level":"info","ts":"2026-07-10T10:49:37.018Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"cab51251-26bc-47b6-a7e7-ffc3702112ab"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"941c73a3-547a-469f-b4ae-964689981fa7"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"a6cf82c5-f605-4d0f-a1f3-fd9d8aedcc78","rayCluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"reconcileID":"5b034d73-caf4-45fe-b36b-393563acf4a4"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"b7225646-76f6-4304-825f-0ae38cc54997"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"a6cf82c5-f605-4d0f-a1f3-fd9d8aedcc78","pod":"fail-4h4dv-head-vxmcp","ip":"10.132.0.31"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"a6cf82c5-f605-4d0f-a1f3-fd9d8aedcc78","pod":"fail-4h4dv-small-group-worker-546nz","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-10T10:49:37.024Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"a6cf82c5-f605-4d0f-a1f3-fd9d8aedcc78","rayCluster":"fail-4h4dv","podIPs":["10.132.0.31","10.133.0.42"]}
{"level":"info","ts":"2026-07-10T10:49:37.031Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:37.037Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","name":"test-ns-z4qpg-fail-4h4dv","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:49:37.042Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","configMap":"kube-rbac-proxy-config-fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.047Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","serviceAccount":"fail-4h4dv-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:49:37.068Z","logger":"raycluster-resource","msg":"default","name":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.068Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-4h4dv","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"530eb39e-ea31-460e-a97b-a9afee3ef01e","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea","cluster":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"237f2c35-27e1-49ab-97ce-7481af3ef836","namespace":"test-ns-z4qpg","clusterName":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"edb5d28b-0095-4d95-a300-de5c8a41bdc3"}
{"level":"info","ts":"2026-07-10T10:49:37.081Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"fab17cca-fef8-49b1-8844-26d7449221eb"}
{"level":"info","ts":"2026-07-10T10:49:37.087Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"dde492e2-e5c3-4dfc-9a21-7ef97252a5c6"}
{"level":"error","ts":"2026-07-10T10:49:37.125Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea","error":"rayclusters.ray.io \"fail-4h4dv\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:37.125Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea"}
{"level":"error","ts":"2026-07-10T10:49:37.125Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"df5fa26c-1343-4365-96df-91887ea51eea","error":"rayclusters.ray.io \"fail-4h4dv\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:37.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c6402b9f-7bf6-4559-bc70-0177acffae17","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:37.125Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c6402b9f-7bf6-4559-bc70-0177acffae17"}
{"level":"info","ts":"2026-07-10T10:49:37.125Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c6402b9f-7bf6-4559-bc70-0177acffae17","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:49:37.125Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c6402b9f-7bf6-4559-bc70-0177acffae17","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:37.125Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"c86a8937-0996-45fa-8d02-201fa93c8cb1"}
{"level":"error","ts":"2026-07-10T10:49:37.128Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"237f2c35-27e1-49ab-97ce-7481af3ef836","issuer":"ray-selfsigned-issuer-fail-4h4dv","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-4h4dv\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:37.130Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3809d3dc-f7d5-4e89-905c-a9ce9df1c062","namespacedName":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:37.130Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3809d3dc-f7d5-4e89-905c-a9ce9df1c062"}
{"level":"info","ts":"2026-07-10T10:49:37.130Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3809d3dc-f7d5-4e89-905c-a9ce9df1c062","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:49:37.130Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"3809d3dc-f7d5-4e89-905c-a9ce9df1c062","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:37.225Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-z4qpg"},"reconcileID":"521685a6-2090-41b2-94a6-5d358dba9f72"}
{"level":"info","ts":"2026-07-10T10:49:37.338Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"237f2c35-27e1-49ab-97ce-7481af3ef836","errorCount":1}
{"level":"info","ts":"2026-07-10T10:49:37.338Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"237f2c35-27e1-49ab-97ce-7481af3ef836"}
{"level":"error","ts":"2026-07-10T10:49:37.338Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"237f2c35-27e1-49ab-97ce-7481af3ef836","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-4h4dv\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:37.343Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"88134202-0b6d-48aa-9a00-ab2c4d5d6dbc","namespace":"test-ns-z4qpg","clusterName":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:37.343Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-4h4dv","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-4h4dv","reconcileID":"88134202-0b6d-48aa-9a00-ab2c4d5d6dbc","namespace":"test-ns-z4qpg","clusterName":"fail-4h4dv"}
{"level":"info","ts":"2026-07-10T10:49:38.002Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d0fa6fc8-e2fd-4606-9bb9-565cb3e8959b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:38.002Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d0fa6fc8-e2fd-4606-9bb9-565cb3e8959b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:49:38.014Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d0fa6fc8-e2fd-4606-9bb9-565cb3e8959b"}
{"level":"info","ts":"2026-07-10T10:49:38.014Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d0fa6fc8-e2fd-4606-9bb9-565cb3e8959b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:49:38.014Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d0fa6fc8-e2fd-4606-9bb9-565cb3e8959b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-26l4b","rayClusterName":"fail-k8s-job-gpvmn","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:49:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:49:38.014Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d0fa6fc8-e2fd-4606-9bb9-565cb3e8959b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:49:38.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"276f950d-78e7-4962-be2f-f5c55458bd6d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:38.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"276f950d-78e7-4962-be2f-f5c55458bd6d","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.026Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"276f950d-78e7-4962-be2f-f5c55458bd6d","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.026Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"276f950d-78e7-4962-be2f-f5c55458bd6d","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.030Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.030Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"276f950d-78e7-4962-be2f-f5c55458bd6d","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"276f950d-78e7-4962-be2f-f5c55458bd6d"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"276f950d-78e7-4962-be2f-f5c55458bd6d","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"fb36c74f-c6b5-4a2d-9170-38fa94f1ca99","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","mode":"IntegratedOAuth","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7058a0a5-5bb3-4d53-9551-2f33acd3664d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7058a0a5-5bb3-4d53-9551-2f33acd3664d","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7058a0a5-5bb3-4d53-9551-2f33acd3664d","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7058a0a5-5bb3-4d53-9551-2f33acd3664d"}
{"level":"info","ts":"2026-07-10T10:49:38.046Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7058a0a5-5bb3-4d53-9551-2f33acd3664d","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:38.047Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"763b8fdd-cebf-4279-a5be-50772392d757","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:38.047Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"763b8fdd-cebf-4279-a5be-50772392d757","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.047Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"763b8fdd-cebf-4279-a5be-50772392d757","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.047Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"763b8fdd-cebf-4279-a5be-50772392d757"}
{"level":"info","ts":"2026-07-10T10:49:38.047Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"763b8fdd-cebf-4279-a5be-50772392d757","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:38.053Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.053Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","name":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:49:38.057Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"fb36c74f-c6b5-4a2d-9170-38fa94f1ca99","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","name":"fail-k8s-job-gpvmn","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-gpvmn\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn"}}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"error","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"303a7842-1ff0-4d71-88cf-d33dbc90f6f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"cb8f66b4-6fcd-486e-8e60-68bc408fb3b4","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:49:38.125Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:38.126Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3edcf30c-2e06-4b95-acfc-b001003d8257","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:38.126Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3edcf30c-2e06-4b95-acfc-b001003d8257","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.126Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3edcf30c-2e06-4b95-acfc-b001003d8257","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.126Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3edcf30c-2e06-4b95-acfc-b001003d8257"}
{"level":"info","ts":"2026-07-10T10:49:38.126Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3edcf30c-2e06-4b95-acfc-b001003d8257","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:38.135Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","name":"fail-k8s-job-gpvmn-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:49:38.135Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:49:38.135Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","name":"fail-k8s-job-gpvmn","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-gpvmn\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn"}}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7"}
{"level":"error","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"0c2050a5-0476-4eac-ae5d-ff1e8b71ffe7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0"}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0"}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:49:38.136Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:38.142Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"cb8f66b4-6fcd-486e-8e60-68bc408fb3b4","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:38.143Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg","operation":"created"}
{"level":"info","ts":"2026-07-10T10:49:38.143Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0"}
{"level":"error","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3c80d876-0eff-4a82-84d6-5f8c13378ab0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"66c9a366-a6b1-475a-a675-a84a39b0c7ab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"66c9a366-a6b1-475a-a675-a84a39b0c7ab","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.148Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"66c9a366-a6b1-475a-a675-a84a39b0c7ab","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.149Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"66c9a366-a6b1-475a-a675-a84a39b0c7ab"}
{"level":"info","ts":"2026-07-10T10:49:38.149Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"66c9a366-a6b1-475a-a675-a84a39b0c7ab","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:38.225Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:49:38.225Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","name":"test-ns-z4qpg-fail-k8s-job-gpvmn","operation":"created"}
{"level":"info","ts":"2026-07-10T10:49:38.230Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"cb8f66b4-6fcd-486e-8e60-68bc408fb3b4","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:38.230Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"cb8f66b4-6fcd-486e-8e60-68bc408fb3b4","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.230Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"31982f28-99fd-4d67-b05b-ede20466cc27","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.232Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","name":"kube-rbac-proxy-config-fail-k8s-job-gpvmn","operation":"created"}
{"level":"info","ts":"2026-07-10T10:49:38.233Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","name":"fail-k8s-job-gpvmn","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-gpvmn\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn"}}
{"level":"info","ts":"2026-07-10T10:49:38.233Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46"}
{"level":"error","ts":"2026-07-10T10:49:38.233Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"3b529b4a-f478-4933-99e0-055e0e37cf46","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.233Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240"}
{"level":"info","ts":"2026-07-10T10:49:38.233Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240"}
{"level":"info","ts":"2026-07-10T10:49:38.233Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240"}
{"level":"error","ts":"2026-07-10T10:49:38.234Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fe98cabb-1bfd-4e1b-8c32-35bb67d15240","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b","rayCluster":"fail-k8s-job-gpvmn","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"dfbf42cd-c5b7-4f69-b68c-d4f59e51bc0b"}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"40b0a420-7d3d-43f8-9999-a33570ff0d97","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.234Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"40b0a420-7d3d-43f8-9999-a33570ff0d97"}
{"level":"error","ts":"2026-07-10T10:49:38.238Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-gpvmn\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.238Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a"}
{"level":"error","ts":"2026-07-10T10:49:38.238Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6f290adb-210c-4fb4-aa54-61629d63bf3a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-gpvmn\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.238Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"78723897-e46b-4788-89ae-ca39bc157645","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.238Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"78723897-e46b-4788-89ae-ca39bc157645","mode":"IntegratedOAuth","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.238Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"78723897-e46b-4788-89ae-ca39bc157645","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.244Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"31982f28-99fd-4d67-b05b-ede20466cc27","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5"}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5"}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5"}
{"level":"error","ts":"2026-07-10T10:49:38.325Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"1ec3b700-b1e0-40d5-9253-6d8b075fa9d5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.329Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"31982f28-99fd-4d67-b05b-ede20466cc27","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:38.329Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"31982f28-99fd-4d67-b05b-ede20466cc27","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"78723897-e46b-4788-89ae-ca39bc157645","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"a650a60c-9d98-4101-9914-9b6cf9fbc63b","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"a650a60c-9d98-4101-9914-9b6cf9fbc63b","mode":"IntegratedOAuth","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"a650a60c-9d98-4101-9914-9b6cf9fbc63b","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"555334a8-dcd9-4368-85eb-e31f868e269b","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"dc79da13-c4f6-4645-aa83-95135b3058b3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"dc79da13-c4f6-4645-aa83-95135b3058b3","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"555334a8-dcd9-4368-85eb-e31f868e269b"}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"dc79da13-c4f6-4645-aa83-95135b3058b3","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.336Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ec6ff744-49d8-4df7-aa83-39b15e0b8214","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"dc79da13-c4f6-4645-aa83-95135b3058b3"}
{"level":"info","ts":"2026-07-10T10:49:38.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"dc79da13-c4f6-4645-aa83-95135b3058b3","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"error","ts":"2026-07-10T10:49:38.425Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"a650a60c-9d98-4101-9914-9b6cf9fbc63b","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-gpvmn\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.425Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"a650a60c-9d98-4101-9914-9b6cf9fbc63b"}
{"level":"error","ts":"2026-07-10T10:49:38.425Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"a650a60c-9d98-4101-9914-9b6cf9fbc63b","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-gpvmn\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.430Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"79095866-0d6d-4deb-aa0b-12dd62313eef","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:38.430Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"79095866-0d6d-4deb-aa0b-12dd62313eef","mode":"IntegratedOAuth","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.430Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"79095866-0d6d-4deb-aa0b-12dd62313eef","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.438Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ec6ff744-49d8-4df7-aa83-39b15e0b8214","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:38.439Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"79095866-0d6d-4deb-aa0b-12dd62313eef","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.449Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ec6ff744-49d8-4df7-aa83-39b15e0b8214","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:38.449Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ec6ff744-49d8-4df7-aa83-39b15e0b8214","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b"}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b"}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b"}
{"level":"error","ts":"2026-07-10T10:49:38.486Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"d49c00b3-7de2-4ba3-9ce0-3333fe24d61b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.807Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213"}
{"level":"info","ts":"2026-07-10T10:49:38.807Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213"}
{"level":"info","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213"}
{"level":"error","ts":"2026-07-10T10:49:38.808Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"45c130b1-c5d9-487c-915b-a77dbf0d4213","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5"}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5"}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","error":"head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"error","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found"}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5"}
{"level":"error","ts":"2026-07-10T10:49:39.449Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9daa9999-90fe-441d-986a-1858164249e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-gpvmn not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","mode":"IntegratedOAuth","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","cluster":"fail-k8s-job-gpvmn","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","cluster":"fail-k8s-job-gpvmn","authType":"IntegratedOAuth","serviceAccount":"fail-k8s-job-gpvmn-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-k8s-job-gpvmn","ray.io/group":"headgroup","ray.io/identifier":"fail-k8s-job-gpvmn-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:49:40.730Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","enabled":true}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","name":"fail-k8s-job-gpvmn-head-h5vrf"}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6"}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","nodeType":"worker","rayStartParams":{"address":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:49:40.747Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","rayStartCmd":"ray start  --address=fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:49:40.748Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:49:40.748Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","enabled":true}
{"level":"info","ts":"2026-07-10T10:49:40.775Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","name":"fail-k8s-job-gpvmn-small-group-worker-lplvs"}
{"level":"info","ts":"2026-07-10T10:49:40.775Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.775Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.775Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-gpvmn-head-svc"},"newHead":{"podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"}}
{"level":"info","ts":"2026-07-10T10:49:40.775Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"158ea2ac-3000-4404-841e-f68fdc85f3f6","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6368ca02-19d5-4fc0-aa7f-5d28c19c6bb3","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"5b67a4f8-487c-40e7-90df-a81378c03a93","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"6368ca02-19d5-4fc0-aa7f-5d28c19c6bb3","rayCluster":"fail-k8s-job-gpvmn","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"de3bcb5c-7112-44e5-983d-acd5e31e53c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"de3bcb5c-7112-44e5-983d-acd5e31e53c6","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"de3bcb5c-7112-44e5-983d-acd5e31e53c6","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:40.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"de3bcb5c-7112-44e5-983d-acd5e31e53c6"}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"de3bcb5c-7112-44e5-983d-acd5e31e53c6","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:40.792Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7219ba9d-2b4f-42b9-8ec0-eda5797c1179","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:49:40.799Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"31e12c15-6d83-4d23-8201-8a945520d5c7","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"70042813-d870-4d0d-a1d3-0bab19bd96da","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"ba158a53-4774-430b-8be7-7427d8bce419","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"ba158a53-4774-430b-8be7-7427d8bce419","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"ba158a53-4774-430b-8be7-7427d8bce419","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.825Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"70042813-d870-4d0d-a1d3-0bab19bd96da","rayCluster":"fail-k8s-job-gpvmn","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:49:40.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.826Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"ba158a53-4774-430b-8be7-7427d8bce419"}
{"level":"info","ts":"2026-07-10T10:49:40.826Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"ba158a53-4774-430b-8be7-7427d8bce419","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:40.826Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:40.826Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"84bfd2cf-c3a7-4116-ad67-bb9f03658d2c","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"5b67a4f8-487c-40e7-90df-a81378c03a93","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:40.831Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c644b8a2-5adc-4696-8227-254bb1f6d43b","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:40.846Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"5b67a4f8-487c-40e7-90df-a81378c03a93","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:40.846Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"5b67a4f8-487c-40e7-90df-a81378c03a93","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.846Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"317595d1-dcdc-4b6b-a53e-027167e787c4","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:40.860Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"317595d1-dcdc-4b6b-a53e-027167e787c4","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:40.875Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"317595d1-dcdc-4b6b-a53e-027167e787c4","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:40.875Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"317595d1-dcdc-4b6b-a53e-027167e787c4","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:41.027Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1b05e1d5-c31a-4038-bd49-06aa76678bba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:41.027Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1b05e1d5-c31a-4038-bd49-06aa76678bba","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:41.027Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1b05e1d5-c31a-4038-bd49-06aa76678bba","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:41.027Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1b05e1d5-c31a-4038-bd49-06aa76678bba"}
{"level":"info","ts":"2026-07-10T10:49:41.027Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1b05e1d5-c31a-4038-bd49-06aa76678bba","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:41.213Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"581a8fb5-ea95-46b6-a945-66e1a8be2d31","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:41.234Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"969023ec-d067-4808-a86c-e1cb9dfaca97","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:41.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"newHead":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"}}
{"level":"info","ts":"2026-07-10T10:49:41.439Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"c4efc3b5-1455-49be-aaf4-14346fe35179","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"04de026c-144b-4e20-9301-4537e31f2fc8","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"336c85a8-cd02-4655-a093-df41a8c39502","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"59f075a2-3a03-42d8-895c-9c1a45d351d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"59f075a2-3a03-42d8-895c-9c1a45d351d5","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"04de026c-144b-4e20-9301-4537e31f2fc8","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"59f075a2-3a03-42d8-895c-9c1a45d351d5","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"59f075a2-3a03-42d8-895c-9c1a45d351d5"}
{"level":"info","ts":"2026-07-10T10:49:41.453Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"59f075a2-3a03-42d8-895c-9c1a45d351d5","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:41.467Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"04de026c-144b-4e20-9301-4537e31f2fc8","certificate":"ray-head-cert-fail-k8s-job-gpvmn","dnsNames":["fail-k8s-job-gpvmn-head-svc","localhost","fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc","fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:49:41.469Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"336c85a8-cd02-4655-a093-df41a8c39502","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:41.481Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"336c85a8-cd02-4655-a093-df41a8c39502","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:41.481Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"336c85a8-cd02-4655-a093-df41a8c39502","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:41.482Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"04de026c-144b-4e20-9301-4537e31f2fc8","certificate":"ray-worker-cert-fail-k8s-job-gpvmn","dnsNames":["fail-k8s-job-gpvmn-worker-svc","localhost","fail-k8s-job-gpvmn-small-group","fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc","fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc.cluster.local","*.fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc","*.fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc.cluster.local","*-worker-*.test-ns-z4qpg.svc","*-worker-*.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:49:41.482Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"04de026c-144b-4e20-9301-4537e31f2fc8","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.132.0.32"]}
{"level":"info","ts":"2026-07-10T10:49:42.205Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9"}
{"level":"info","ts":"2026-07-10T10:49:42.205Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9"}
{"level":"info","ts":"2026-07-10T10:49:42.205Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:42.206Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"02b3866c-0b88-42af-98ad-f795196d5fb9","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5"}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5"}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:42.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:42.793Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:42.793Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"2f1bcae8-1b68-4ae0-8d7b-3d6db28b56b5","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:44.028Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"19637633-293e-495f-941d-10635403b496","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:44.028Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"19637633-293e-495f-941d-10635403b496","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:44.028Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"19637633-293e-495f-941d-10635403b496","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:44.028Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"19637633-293e-495f-941d-10635403b496"}
{"level":"info","ts":"2026-07-10T10:49:44.028Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"19637633-293e-495f-941d-10635403b496","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:46.455Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449"}
{"level":"info","ts":"2026-07-10T10:49:46.455Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449"}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:46.456Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"10879ef0-283b-4b24-a6ea-bcaaf5705449","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:47.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"03bf657a-4fc1-4dcb-b4c7-505d3f2338e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:47.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"03bf657a-4fc1-4dcb-b4c7-505d3f2338e7","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:47.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"03bf657a-4fc1-4dcb-b4c7-505d3f2338e7","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:47.029Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"03bf657a-4fc1-4dcb-b4c7-505d3f2338e7"}
{"level":"info","ts":"2026-07-10T10:49:47.029Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"03bf657a-4fc1-4dcb-b4c7-505d3f2338e7","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:47.465Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3"}
{"level":"info","ts":"2026-07-10T10:49:47.465Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3"}
{"level":"info","ts":"2026-07-10T10:49:47.465Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:47.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:49:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","podName":"fail-k8s-job-gpvmn-head-h5vrf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:49:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:49:47.466Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"663c7e37-9530-4a58-82de-c5f34d844cb3","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ff2aac82-27d8-478e-a77e-83e50e628e53","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ac1e30ed-3cdc-4d87-a810-0325e63422b9","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ff2aac82-27d8-478e-a77e-83e50e628e53","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ff2aac82-27d8-478e-a77e-83e50e628e53","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d314086e-60f5-4b36-ae13-e635e11614b8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d314086e-60f5-4b36-ae13-e635e11614b8","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d314086e-60f5-4b36-ae13-e635e11614b8","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d314086e-60f5-4b36-ae13-e635e11614b8"}
{"level":"info","ts":"2026-07-10T10:49:47.480Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d314086e-60f5-4b36-ae13-e635e11614b8","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:47.493Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ac1e30ed-3cdc-4d87-a810-0325e63422b9","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:47.493Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ff2aac82-27d8-478e-a77e-83e50e628e53","certificate":"ray-head-cert-fail-k8s-job-gpvmn","dnsNames":["fail-k8s-job-gpvmn-head-svc","localhost","fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc","fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.32","10.133.0.44","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:49:47.505Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ac1e30ed-3cdc-4d87-a810-0325e63422b9","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:47.505Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ac1e30ed-3cdc-4d87-a810-0325e63422b9","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:47.505Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ff2aac82-27d8-478e-a77e-83e50e628e53","certificate":"ray-worker-cert-fail-k8s-job-gpvmn","dnsNames":["fail-k8s-job-gpvmn-worker-svc","localhost","fail-k8s-job-gpvmn-small-group","fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc","fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc.cluster.local","*.fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc","*.fail-k8s-job-gpvmn-worker-svc.test-ns-z4qpg.svc.cluster.local","*-worker-*.test-ns-z4qpg.svc","*-worker-*.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.32","10.133.0.44","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:49:47.505Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ff2aac82-27d8-478e-a77e-83e50e628e53","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.132.0.32","10.133.0.44"]}
{"level":"info","ts":"2026-07-10T10:49:49.480Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001"}
{"level":"info","ts":"2026-07-10T10:49:49.480Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001"}
{"level":"info","ts":"2026-07-10T10:49:49.480Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:49:49.481Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"7b8ba760-a3d8-4cda-9302-c5586b483001","seconds":300}
{"level":"info","ts":"2026-07-10T10:49:50.030Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"eb7ed7ad-fd20-496b-923b-0d45ea2ac8ed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:50.030Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"eb7ed7ad-fd20-496b-923b-0d45ea2ac8ed","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:50.030Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"eb7ed7ad-fd20-496b-923b-0d45ea2ac8ed","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:50.031Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"eb7ed7ad-fd20-496b-923b-0d45ea2ac8ed"}
{"level":"info","ts":"2026-07-10T10:49:50.031Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"eb7ed7ad-fd20-496b-923b-0d45ea2ac8ed","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:53.032Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c6d255c6-2b55-476b-bb75-a76d2578f6b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:53.032Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c6d255c6-2b55-476b-bb75-a76d2578f6b6","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:53.032Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c6d255c6-2b55-476b-bb75-a76d2578f6b6","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:53.032Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c6d255c6-2b55-476b-bb75-a76d2578f6b6"}
{"level":"info","ts":"2026-07-10T10:49:53.032Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c6d255c6-2b55-476b-bb75-a76d2578f6b6","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:56.033Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8d4ead05-74a5-4eb7-9139-da8b59ef8f56","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:56.033Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8d4ead05-74a5-4eb7-9139-da8b59ef8f56","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:56.033Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8d4ead05-74a5-4eb7-9139-da8b59ef8f56","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:56.033Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8d4ead05-74a5-4eb7-9139-da8b59ef8f56"}
{"level":"info","ts":"2026-07-10T10:49:56.033Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8d4ead05-74a5-4eb7-9139-da8b59ef8f56","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:58.495Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d"}
{"level":"info","ts":"2026-07-10T10:49:58.495Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d"}
{"level":"info","ts":"2026-07-10T10:49:58.495Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:49:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:49:58.496Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"171a82ad-2550-4aa9-978a-4740b4500e4d","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:49:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"99a0f623-1b0b-4588-b298-b1633ba6ba8a","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"26948587-1101-4c46-bf0b-58877365057e","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64bc5dec-3171-4f9f-be54-819e1e1363f0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64bc5dec-3171-4f9f-be54-819e1e1363f0","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64bc5dec-3171-4f9f-be54-819e1e1363f0","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"26948587-1101-4c46-bf0b-58877365057e","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"26948587-1101-4c46-bf0b-58877365057e","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"26948587-1101-4c46-bf0b-58877365057e","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.133.0.44","10.132.0.32"]}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64bc5dec-3171-4f9f-be54-819e1e1363f0"}
{"level":"info","ts":"2026-07-10T10:49:58.509Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64bc5dec-3171-4f9f-be54-819e1e1363f0","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:49:58.527Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"99a0f623-1b0b-4588-b298-b1633ba6ba8a","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:49:58.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"99a0f623-1b0b-4588-b298-b1633ba6ba8a","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:49:58.536Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"99a0f623-1b0b-4588-b298-b1633ba6ba8a","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:59.034Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b845a4bf-d1c1-4e1e-aa4a-0b3e46a087e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:49:59.034Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b845a4bf-d1c1-4e1e-aa4a-0b3e46a087e2","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:49:59.034Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b845a4bf-d1c1-4e1e-aa4a-0b3e46a087e2","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:49:59.034Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b845a4bf-d1c1-4e1e-aa4a-0b3e46a087e2"}
{"level":"info","ts":"2026-07-10T10:49:59.034Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b845a4bf-d1c1-4e1e-aa4a-0b3e46a087e2","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629"}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629"}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:50:00.509Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:50:00.510Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:50:00.510Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:50:00.510Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:50:00.510Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"4fc4b36d-ed4f-4108-bdcd-e6b475f56629","seconds":300}
{"level":"info","ts":"2026-07-10T10:50:02.034Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7fd8dc0d-ee91-47da-b542-a2e3424d3de3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:02.034Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7fd8dc0d-ee91-47da-b542-a2e3424d3de3","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:02.034Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7fd8dc0d-ee91-47da-b542-a2e3424d3de3","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:02.035Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7fd8dc0d-ee91-47da-b542-a2e3424d3de3"}
{"level":"info","ts":"2026-07-10T10:50:02.035Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7fd8dc0d-ee91-47da-b542-a2e3424d3de3","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:05.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1df072ab-e034-4039-8e65-09a1f82d431e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:05.036Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1df072ab-e034-4039-8e65-09a1f82d431e","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:05.036Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1df072ab-e034-4039-8e65-09a1f82d431e","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:05.036Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1df072ab-e034-4039-8e65-09a1f82d431e"}
{"level":"info","ts":"2026-07-10T10:50:05.036Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"1df072ab-e034-4039-8e65-09a1f82d431e","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:08.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e06f4365-651e-41d2-81c6-9678122ebd82","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:08.036Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e06f4365-651e-41d2-81c6-9678122ebd82","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:08.036Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e06f4365-651e-41d2-81c6-9678122ebd82","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:08.037Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e06f4365-651e-41d2-81c6-9678122ebd82"}
{"level":"info","ts":"2026-07-10T10:50:08.037Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e06f4365-651e-41d2-81c6-9678122ebd82","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:08.235Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"522992dc-0e41-404c-8444-9ae68e27f611","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:08.235Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"522992dc-0e41-404c-8444-9ae68e27f611","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:50:08.235Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"522992dc-0e41-404c-8444-9ae68e27f611","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:50:08.235Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"522992dc-0e41-404c-8444-9ae68e27f611","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.132.0.32","10.133.0.44"]}
{"level":"info","ts":"2026-07-10T10:50:10.430Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4e878e8a-c14d-4661-b2ae-f641cf674c50","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:50:10.430Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4e878e8a-c14d-4661-b2ae-f641cf674c50","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:50:10.430Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4e878e8a-c14d-4661-b2ae-f641cf674c50","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:50:10.430Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4e878e8a-c14d-4661-b2ae-f641cf674c50","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:50:11.037Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85246769-e18a-47aa-a8df-d6964681f6fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:11.037Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85246769-e18a-47aa-a8df-d6964681f6fe","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:11.037Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85246769-e18a-47aa-a8df-d6964681f6fe","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:11.037Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85246769-e18a-47aa-a8df-d6964681f6fe"}
{"level":"info","ts":"2026-07-10T10:50:11.037Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85246769-e18a-47aa-a8df-d6964681f6fe","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:14.038Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8ceaffb1-3dab-49e6-aa93-6832b87585f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:14.038Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8ceaffb1-3dab-49e6-aa93-6832b87585f7","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:14.038Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8ceaffb1-3dab-49e6-aa93-6832b87585f7","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:14.039Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8ceaffb1-3dab-49e6-aa93-6832b87585f7"}
{"level":"info","ts":"2026-07-10T10:50:14.039Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8ceaffb1-3dab-49e6-aa93-6832b87585f7","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:17.039Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"569486a2-3118-4a47-a660-755f5f1d91df","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:17.039Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"569486a2-3118-4a47-a660-755f5f1d91df","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:17.040Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"569486a2-3118-4a47-a660-755f5f1d91df","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:17.040Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"569486a2-3118-4a47-a660-755f5f1d91df"}
{"level":"info","ts":"2026-07-10T10:50:17.040Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"569486a2-3118-4a47-a660-755f5f1d91df","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:20.040Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0a02ed4e-95ba-4d87-b504-b709ce8314b8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:20.040Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0a02ed4e-95ba-4d87-b504-b709ce8314b8","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:20.040Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0a02ed4e-95ba-4d87-b504-b709ce8314b8","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:20.040Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0a02ed4e-95ba-4d87-b504-b709ce8314b8"}
{"level":"info","ts":"2026-07-10T10:50:20.040Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0a02ed4e-95ba-4d87-b504-b709ce8314b8","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:23.041Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f318d262-a091-4eb1-af1f-7fe0d7538385","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:23.041Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f318d262-a091-4eb1-af1f-7fe0d7538385","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:23.041Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f318d262-a091-4eb1-af1f-7fe0d7538385","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:23.041Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f318d262-a091-4eb1-af1f-7fe0d7538385"}
{"level":"info","ts":"2026-07-10T10:50:23.041Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f318d262-a091-4eb1-af1f-7fe0d7538385","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:26.042Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fc0b9c1a-f8cf-455c-b702-5625775ee380","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:26.042Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fc0b9c1a-f8cf-455c-b702-5625775ee380","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:26.042Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fc0b9c1a-f8cf-455c-b702-5625775ee380","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:26.042Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fc0b9c1a-f8cf-455c-b702-5625775ee380"}
{"level":"info","ts":"2026-07-10T10:50:26.042Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fc0b9c1a-f8cf-455c-b702-5625775ee380","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:29.043Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bdb1fc00-ac90-453d-a436-6b05b0a5ad04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:29.043Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bdb1fc00-ac90-453d-a436-6b05b0a5ad04","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:29.043Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bdb1fc00-ac90-453d-a436-6b05b0a5ad04","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:29.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bdb1fc00-ac90-453d-a436-6b05b0a5ad04"}
{"level":"info","ts":"2026-07-10T10:50:29.043Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bdb1fc00-ac90-453d-a436-6b05b0a5ad04","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:32.044Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6555250a-b3e8-4b6a-9d91-77923b90216a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:32.044Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6555250a-b3e8-4b6a-9d91-77923b90216a","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:32.044Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6555250a-b3e8-4b6a-9d91-77923b90216a","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:32.044Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6555250a-b3e8-4b6a-9d91-77923b90216a"}
{"level":"info","ts":"2026-07-10T10:50:32.044Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6555250a-b3e8-4b6a-9d91-77923b90216a","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:35.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c835b487-4181-4f59-9f80-bad920a02671","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:35.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c835b487-4181-4f59-9f80-bad920a02671","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:35.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c835b487-4181-4f59-9f80-bad920a02671","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:35.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c835b487-4181-4f59-9f80-bad920a02671"}
{"level":"info","ts":"2026-07-10T10:50:35.046Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c835b487-4181-4f59-9f80-bad920a02671","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:38.047Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d22f715b-b59c-4fde-a8a0-064e5e7c0a78","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:38.047Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d22f715b-b59c-4fde-a8a0-064e5e7c0a78","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:38.047Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d22f715b-b59c-4fde-a8a0-064e5e7c0a78","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:38.047Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d22f715b-b59c-4fde-a8a0-064e5e7c0a78"}
{"level":"info","ts":"2026-07-10T10:50:38.047Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d22f715b-b59c-4fde-a8a0-064e5e7c0a78","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:41.048Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"39d609d5-45da-40ff-a042-5b4254b76ce0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:41.048Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"39d609d5-45da-40ff-a042-5b4254b76ce0","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:41.048Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"39d609d5-45da-40ff-a042-5b4254b76ce0","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:41.048Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"39d609d5-45da-40ff-a042-5b4254b76ce0"}
{"level":"info","ts":"2026-07-10T10:50:41.048Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"39d609d5-45da-40ff-a042-5b4254b76ce0","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:44.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85d2c62f-7e02-48aa-9baf-fadaf6bf2232","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:44.049Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85d2c62f-7e02-48aa-9baf-fadaf6bf2232","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:44.049Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85d2c62f-7e02-48aa-9baf-fadaf6bf2232","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:44.049Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85d2c62f-7e02-48aa-9baf-fadaf6bf2232"}
{"level":"info","ts":"2026-07-10T10:50:44.049Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"85d2c62f-7e02-48aa-9baf-fadaf6bf2232","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:47.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a077ab2e-82a1-419a-9511-a46ad50113be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:47.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a077ab2e-82a1-419a-9511-a46ad50113be","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:47.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a077ab2e-82a1-419a-9511-a46ad50113be","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:47.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a077ab2e-82a1-419a-9511-a46ad50113be"}
{"level":"info","ts":"2026-07-10T10:50:47.050Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a077ab2e-82a1-419a-9511-a46ad50113be","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:50.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"99e16989-e058-47f7-b0bc-c2e1b398e694","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:50.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"99e16989-e058-47f7-b0bc-c2e1b398e694","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:50.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"99e16989-e058-47f7-b0bc-c2e1b398e694","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:50.051Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"99e16989-e058-47f7-b0bc-c2e1b398e694"}
{"level":"info","ts":"2026-07-10T10:50:50.051Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"99e16989-e058-47f7-b0bc-c2e1b398e694","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:53.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"86789747-da95-403e-9710-4136d8e0940e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:53.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"86789747-da95-403e-9710-4136d8e0940e","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:53.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"86789747-da95-403e-9710-4136d8e0940e","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:53.052Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"86789747-da95-403e-9710-4136d8e0940e"}
{"level":"info","ts":"2026-07-10T10:50:53.052Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"86789747-da95-403e-9710-4136d8e0940e","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:56.053Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a59d179e-2458-4ef7-95d6-c27f7f77ae19","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:56.053Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a59d179e-2458-4ef7-95d6-c27f7f77ae19","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:56.053Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a59d179e-2458-4ef7-95d6-c27f7f77ae19","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:56.053Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a59d179e-2458-4ef7-95d6-c27f7f77ae19"}
{"level":"info","ts":"2026-07-10T10:50:56.053Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a59d179e-2458-4ef7-95d6-c27f7f77ae19","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:50:59.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e4da8256-6eef-4809-aeed-35230e934182","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:50:59.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e4da8256-6eef-4809-aeed-35230e934182","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:50:59.054Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e4da8256-6eef-4809-aeed-35230e934182","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:50:59.054Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e4da8256-6eef-4809-aeed-35230e934182"}
{"level":"info","ts":"2026-07-10T10:50:59.054Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e4da8256-6eef-4809-aeed-35230e934182","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:02.055Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c327fe8-1a38-4655-982e-f9a72c2bae2e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:02.055Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c327fe8-1a38-4655-982e-f9a72c2bae2e","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:02.055Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c327fe8-1a38-4655-982e-f9a72c2bae2e","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:02.055Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c327fe8-1a38-4655-982e-f9a72c2bae2e"}
{"level":"info","ts":"2026-07-10T10:51:02.055Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c327fe8-1a38-4655-982e-f9a72c2bae2e","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:05.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b13c5073-50e0-4156-988a-0b370dbbde80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:05.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b13c5073-50e0-4156-988a-0b370dbbde80","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:05.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b13c5073-50e0-4156-988a-0b370dbbde80","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:05.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b13c5073-50e0-4156-988a-0b370dbbde80"}
{"level":"info","ts":"2026-07-10T10:51:05.056Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b13c5073-50e0-4156-988a-0b370dbbde80","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3"}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3"}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:07.588Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:51:07.589Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:51:07.589Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:51:07.589Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"fd7385d6-1715-4887-9745-74ea583d83c3","seconds":300}
{"level":"info","ts":"2026-07-10T10:51:08.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"cb1d35e6-38bc-4cf8-b8a6-305c9ab620cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:08.057Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"cb1d35e6-38bc-4cf8-b8a6-305c9ab620cd","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:08.057Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"cb1d35e6-38bc-4cf8-b8a6-305c9ab620cd","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:08.057Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"cb1d35e6-38bc-4cf8-b8a6-305c9ab620cd"}
{"level":"info","ts":"2026-07-10T10:51:08.057Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"cb1d35e6-38bc-4cf8-b8a6-305c9ab620cd","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:08.236Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"7ed51a75-f566-4ad4-bc59-2a58084648f5","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:08.236Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"7ed51a75-f566-4ad4-bc59-2a58084648f5","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:51:08.236Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"7ed51a75-f566-4ad4-bc59-2a58084648f5","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:51:08.236Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"7ed51a75-f566-4ad4-bc59-2a58084648f5","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.132.0.32","10.133.0.44"]}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f"}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f"}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:51:08.596Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:51:08.597Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:51:08.597Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"cf27f9c6-1c25-4ed1-9258-47f118635d3f","seconds":300}
{"level":"info","ts":"2026-07-10T10:51:10.431Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"30334444-585d-4cff-93c6-604cf96a0a79","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:51:10.431Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"30334444-585d-4cff-93c6-604cf96a0a79","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:51:10.431Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"30334444-585d-4cff-93c6-604cf96a0a79","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:51:10.431Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"30334444-585d-4cff-93c6-604cf96a0a79","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:51:11.058Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e9aa0d9d-5b34-45f0-bf1d-e649982eed9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:11.058Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e9aa0d9d-5b34-45f0-bf1d-e649982eed9a","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:11.058Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e9aa0d9d-5b34-45f0-bf1d-e649982eed9a","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:11.058Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e9aa0d9d-5b34-45f0-bf1d-e649982eed9a"}
{"level":"info","ts":"2026-07-10T10:51:11.058Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e9aa0d9d-5b34-45f0-bf1d-e649982eed9a","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:14.059Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f60c55a3-3394-4037-a3b6-d353c3e24c8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:14.059Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f60c55a3-3394-4037-a3b6-d353c3e24c8b","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:14.059Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f60c55a3-3394-4037-a3b6-d353c3e24c8b","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:14.059Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f60c55a3-3394-4037-a3b6-d353c3e24c8b"}
{"level":"info","ts":"2026-07-10T10:51:14.059Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f60c55a3-3394-4037-a3b6-d353c3e24c8b","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:17.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3eb34aa2-294c-49e8-adc2-7ed1013fecc6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:17.060Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3eb34aa2-294c-49e8-adc2-7ed1013fecc6","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:17.060Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3eb34aa2-294c-49e8-adc2-7ed1013fecc6","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:17.060Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3eb34aa2-294c-49e8-adc2-7ed1013fecc6"}
{"level":"info","ts":"2026-07-10T10:51:17.060Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3eb34aa2-294c-49e8-adc2-7ed1013fecc6","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:51:17.636Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"9ddc3bf5-34fb-4dcc-ba07-5660b2d85a24","seconds":300}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6"}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6"}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:49:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:49:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:51:18.641Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"422c4d81-71f4-4ae0-a041-9a9f3c6ae5b6","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:49:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:49:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:51:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:49:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:51:18.653Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f35758fe-7807-4af2-9af2-508d1722c1b6","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:18.653Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"63894f86-4d3b-45c0-80b8-cdb159fc363d","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4bee4040-3e6f-4b5d-b925-90b72e421682","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4bee4040-3e6f-4b5d-b925-90b72e421682","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f35758fe-7807-4af2-9af2-508d1722c1b6","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f35758fe-7807-4af2-9af2-508d1722c1b6","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4bee4040-3e6f-4b5d-b925-90b72e421682","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f35758fe-7807-4af2-9af2-508d1722c1b6","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.132.0.32","10.133.0.44"]}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4bee4040-3e6f-4b5d-b925-90b72e421682"}
{"level":"info","ts":"2026-07-10T10:51:18.654Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4bee4040-3e6f-4b5d-b925-90b72e421682","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:18.669Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"63894f86-4d3b-45c0-80b8-cdb159fc363d","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:51:18.680Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"63894f86-4d3b-45c0-80b8-cdb159fc363d","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:51:18.680Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"63894f86-4d3b-45c0-80b8-cdb159fc363d","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:20.061Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c73ed43d-c25a-4954-8d98-a0ffb8b8ba20","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:20.061Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c73ed43d-c25a-4954-8d98-a0ffb8b8ba20","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:20.061Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c73ed43d-c25a-4954-8d98-a0ffb8b8ba20","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:20.061Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c73ed43d-c25a-4954-8d98-a0ffb8b8ba20"}
{"level":"info","ts":"2026-07-10T10:51:20.061Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"c73ed43d-c25a-4954-8d98-a0ffb8b8ba20","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37"}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37"}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:49:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:51:20.653Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","podName":"fail-k8s-job-gpvmn-small-group-worker-lplvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:49:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:51:20.654Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:51:20.654Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"6726d6dd-0b62-4213-adc9-a59ca5eb0e37","seconds":300}
{"level":"info","ts":"2026-07-10T10:51:23.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b3b14168-68b2-40d3-accb-7a6db79552bd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:23.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b3b14168-68b2-40d3-accb-7a6db79552bd","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:23.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b3b14168-68b2-40d3-accb-7a6db79552bd","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:23.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b3b14168-68b2-40d3-accb-7a6db79552bd"}
{"level":"info","ts":"2026-07-10T10:51:23.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b3b14168-68b2-40d3-accb-7a6db79552bd","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:26.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3d0a98d0-1a4d-4e34-9c61-4fa35b9599e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:26.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3d0a98d0-1a4d-4e34-9c61-4fa35b9599e4","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:26.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3d0a98d0-1a4d-4e34-9c61-4fa35b9599e4","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:26.063Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3d0a98d0-1a4d-4e34-9c61-4fa35b9599e4"}
{"level":"info","ts":"2026-07-10T10:51:26.063Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"3d0a98d0-1a4d-4e34-9c61-4fa35b9599e4","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:29.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5bbd3778-7e1d-4500-9d45-128aa7d6b6cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.063Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5bbd3778-7e1d-4500-9d45-128aa7d6b6cb","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.063Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5bbd3778-7e1d-4500-9d45-128aa7d6b6cb","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.063Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5bbd3778-7e1d-4500-9d45-128aa7d6b6cb"}
{"level":"info","ts":"2026-07-10T10:51:29.063Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5bbd3778-7e1d-4500-9d45-128aa7d6b6cb","RayCluster":"fail-k8s-job-gpvmn","State":""}
{"level":"info","ts":"2026-07-10T10:51:29.667Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48"}
{"level":"info","ts":"2026-07-10T10:51:29.667Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48"}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:51:29.668Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"82682656-8076-45c1-8d6c-058302a48a48","name":"fail-k8s-job-gpvmn","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:51:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:49:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:51:29Z","stateTransitionTimes":{"ready":"2026-07-10T10:51:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-k8s-job-gpvmn-head-h5vrf","serviceName":"fail-k8s-job-gpvmn-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:49:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:51:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:49:38Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:49:38Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"3e2ac326-8018-419e-bc7a-6ccf187ec5d3","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"06ec1fae-169d-4430-8d02-824f47475ded","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"06ec1fae-169d-4430-8d02-824f47475ded","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"06ec1fae-169d-4430-8d02-824f47475ded","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"06ec1fae-169d-4430-8d02-824f47475ded","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.132.0.32","10.133.0.44"]}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","head service name":"fail-k8s-job-gpvmn-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","head service URL":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984"}
{"level":"info","ts":"2026-07-10T10:51:29.682Z","logger":"controllers.RayJob","msg":"User-provided command is used","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","command":["ray","job","submit","--address","http://do-not-exist:8265","--","echo 123"]}
{"level":"info","ts":"2026-07-10T10:51:29.697Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-07-10T10:51:29.697Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.697Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-26l4b","rayClusterName":"fail-k8s-job-gpvmn","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:49:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-26l4b","rayClusterName":"fail-k8s-job-gpvmn","dashboardURL":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:49:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:51:29.697Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"53493644-be02-4b69-9248-9d0630479984","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:51:29.702Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"3e2ac326-8018-419e-bc7a-6ccf187ec5d3","name":"fail-k8s-job-gpvmn-head"}
{"level":"info","ts":"2026-07-10T10:51:29.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.710Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.710Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.711Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740"}
{"level":"info","ts":"2026-07-10T10:51:29.711Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","head service name":"fail-k8s-job-gpvmn-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:51:29.711Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","head service URL":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:51:29.711Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-07-10T10:51:29.711Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.711Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-26l4b","rayClusterName":"fail-k8s-job-gpvmn","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:49:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-26l4b","rayClusterName":"fail-k8s-job-gpvmn","dashboardURL":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:49:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:51:29.711Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:51:29.720Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:51:29.720Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740"}
{"level":"error","ts":"2026-07-10T10:51:29.720Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"5f389ad0-87f0-46cf-8f1a-6c42add12740","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64e80dbd-9298-474f-9577-11174f029bf0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.720Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64e80dbd-9298-474f-9577-11174f029bf0","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.720Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64e80dbd-9298-474f-9577-11174f029bf0","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.720Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64e80dbd-9298-474f-9577-11174f029bf0"}
{"level":"info","ts":"2026-07-10T10:51:29.722Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"3e2ac326-8018-419e-bc7a-6ccf187ec5d3","name":"fail-k8s-job-gpvmn-workers"}
{"level":"info","ts":"2026-07-10T10:51:29.722Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"3e2ac326-8018-419e-bc7a-6ccf187ec5d3","cluster":"fail-k8s-job-gpvmn"}
{"level":"error","ts":"2026-07-10T10:51:29.731Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64e80dbd-9298-474f-9577-11174f029bf0","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.731Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64e80dbd-9298-474f-9577-11174f029bf0"}
{"level":"error","ts":"2026-07-10T10:51:29.731Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"64e80dbd-9298-474f-9577-11174f029bf0","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"37ed9e28-be64-463e-bf2e-2ad9693b75c6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"37ed9e28-be64-463e-bf2e-2ad9693b75c6","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"37ed9e28-be64-463e-bf2e-2ad9693b75c6","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.732Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"37ed9e28-be64-463e-bf2e-2ad9693b75c6"}
{"level":"error","ts":"2026-07-10T10:51:29.737Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"37ed9e28-be64-463e-bf2e-2ad9693b75c6","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.737Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"37ed9e28-be64-463e-bf2e-2ad9693b75c6"}
{"level":"error","ts":"2026-07-10T10:51:29.737Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"37ed9e28-be64-463e-bf2e-2ad9693b75c6","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.742Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"93e1fbb1-7082-4b4f-8861-de51b443b066","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.742Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"93e1fbb1-7082-4b4f-8861-de51b443b066","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.742Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"93e1fbb1-7082-4b4f-8861-de51b443b066","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.742Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"93e1fbb1-7082-4b4f-8861-de51b443b066"}
{"level":"error","ts":"2026-07-10T10:51:29.747Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"93e1fbb1-7082-4b4f-8861-de51b443b066","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.747Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"93e1fbb1-7082-4b4f-8861-de51b443b066"}
{"level":"error","ts":"2026-07-10T10:51:29.747Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"93e1fbb1-7082-4b4f-8861-de51b443b066","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.787Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"820f1789-81d1-42c5-8b77-3bc4d74d2f3b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.787Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"820f1789-81d1-42c5-8b77-3bc4d74d2f3b","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.787Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"820f1789-81d1-42c5-8b77-3bc4d74d2f3b","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.788Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"820f1789-81d1-42c5-8b77-3bc4d74d2f3b"}
{"level":"error","ts":"2026-07-10T10:51:29.793Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"820f1789-81d1-42c5-8b77-3bc4d74d2f3b","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.793Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"820f1789-81d1-42c5-8b77-3bc4d74d2f3b"}
{"level":"error","ts":"2026-07-10T10:51:29.793Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"820f1789-81d1-42c5-8b77-3bc4d74d2f3b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b6888d85-f66d-4ae5-b88b-dbdf91312cf6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:29.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b6888d85-f66d-4ae5-b88b-dbdf91312cf6","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:29.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b6888d85-f66d-4ae5-b88b-dbdf91312cf6","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:29.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b6888d85-f66d-4ae5-b88b-dbdf91312cf6"}
{"level":"error","ts":"2026-07-10T10:51:29.880Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b6888d85-f66d-4ae5-b88b-dbdf91312cf6","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:29.880Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b6888d85-f66d-4ae5-b88b-dbdf91312cf6"}
{"level":"error","ts":"2026-07-10T10:51:29.880Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b6888d85-f66d-4ae5-b88b-dbdf91312cf6","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:30.040Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2d3aeaea-adb5-4c79-a94e-70bee0fd862a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:30.040Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2d3aeaea-adb5-4c79-a94e-70bee0fd862a","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:30.040Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2d3aeaea-adb5-4c79-a94e-70bee0fd862a","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:30.040Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2d3aeaea-adb5-4c79-a94e-70bee0fd862a"}
{"level":"error","ts":"2026-07-10T10:51:30.046Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2d3aeaea-adb5-4c79-a94e-70bee0fd862a","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:30.046Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2d3aeaea-adb5-4c79-a94e-70bee0fd862a"}
{"level":"error","ts":"2026-07-10T10:51:30.046Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2d3aeaea-adb5-4c79-a94e-70bee0fd862a","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:30.366Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fa1232f1-fbb7-4d2b-be83-c4eb3f711ba1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:30.366Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fa1232f1-fbb7-4d2b-be83-c4eb3f711ba1","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:30.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fa1232f1-fbb7-4d2b-be83-c4eb3f711ba1","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:30.367Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fa1232f1-fbb7-4d2b-be83-c4eb3f711ba1"}
{"level":"error","ts":"2026-07-10T10:51:30.372Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fa1232f1-fbb7-4d2b-be83-c4eb3f711ba1","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:30.372Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fa1232f1-fbb7-4d2b-be83-c4eb3f711ba1"}
{"level":"error","ts":"2026-07-10T10:51:30.372Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"fa1232f1-fbb7-4d2b-be83-c4eb3f711ba1","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:30.725Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f931c3f9-7508-40b3-bc61-6660e2df5f5c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:30.725Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f931c3f9-7508-40b3-bc61-6660e2df5f5c","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:30.725Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f931c3f9-7508-40b3-bc61-6660e2df5f5c","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:30.725Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f931c3f9-7508-40b3-bc61-6660e2df5f5c"}
{"level":"error","ts":"2026-07-10T10:51:30.730Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f931c3f9-7508-40b3-bc61-6660e2df5f5c","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:30.730Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f931c3f9-7508-40b3-bc61-6660e2df5f5c"}
{"level":"error","ts":"2026-07-10T10:51:30.730Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"f931c3f9-7508-40b3-bc61-6660e2df5f5c","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:31.013Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e63c9b35-3f97-447c-b428-035690704d4c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:31.013Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e63c9b35-3f97-447c-b428-035690704d4c","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:31.013Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e63c9b35-3f97-447c-b428-035690704d4c","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:31.013Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e63c9b35-3f97-447c-b428-035690704d4c"}
{"level":"error","ts":"2026-07-10T10:51:31.019Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e63c9b35-3f97-447c-b428-035690704d4c","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:31.019Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e63c9b35-3f97-447c-b428-035690704d4c"}
{"level":"error","ts":"2026-07-10T10:51:31.019Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"e63c9b35-3f97-447c-b428-035690704d4c","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d"}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d"}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","1 head service found":"fail-k8s-job-gpvmn-head-svc"}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","Found 1 head Pod":"fail-k8s-job-gpvmn-head-h5vrf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","head Pod":"fail-k8s-job-gpvmn-head-h5vrf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-gpvmn-head-h5vrf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","worker Pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-gpvmn-small-group-worker-lplvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:51:31.683Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"f46d060e-7fd0-45a2-8426-ba40959dde9d","seconds":300}
{"level":"info","ts":"2026-07-10T10:51:33.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bc9eb697-5cc7-4a8f-8bc9-1203ab09ac9a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:33.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bc9eb697-5cc7-4a8f-8bc9-1203ab09ac9a","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:33.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bc9eb697-5cc7-4a8f-8bc9-1203ab09ac9a","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:33.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bc9eb697-5cc7-4a8f-8bc9-1203ab09ac9a"}
{"level":"error","ts":"2026-07-10T10:51:33.586Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bc9eb697-5cc7-4a8f-8bc9-1203ab09ac9a","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:33.586Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bc9eb697-5cc7-4a8f-8bc9-1203ab09ac9a"}
{"level":"error","ts":"2026-07-10T10:51:33.586Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"bc9eb697-5cc7-4a8f-8bc9-1203ab09ac9a","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:35.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"21987ef1-a8f5-44b8-bcf3-7a4deb7605c5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:35.720Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"21987ef1-a8f5-44b8-bcf3-7a4deb7605c5","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:35.720Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"21987ef1-a8f5-44b8-bcf3-7a4deb7605c5","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:35.720Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"21987ef1-a8f5-44b8-bcf3-7a4deb7605c5"}
{"level":"error","ts":"2026-07-10T10:51:35.726Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"21987ef1-a8f5-44b8-bcf3-7a4deb7605c5","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:35.726Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"21987ef1-a8f5-44b8-bcf3-7a4deb7605c5"}
{"level":"error","ts":"2026-07-10T10:51:35.726Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"21987ef1-a8f5-44b8-bcf3-7a4deb7605c5","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:36.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d7e2d677-c8bf-4ba3-8f8a-6f366d9c8004","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:36.728Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d7e2d677-c8bf-4ba3-8f8a-6f366d9c8004","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:36.728Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d7e2d677-c8bf-4ba3-8f8a-6f366d9c8004","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:36.728Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d7e2d677-c8bf-4ba3-8f8a-6f366d9c8004"}
{"level":"error","ts":"2026-07-10T10:51:36.734Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d7e2d677-c8bf-4ba3-8f8a-6f366d9c8004","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:36.734Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d7e2d677-c8bf-4ba3-8f8a-6f366d9c8004"}
{"level":"error","ts":"2026-07-10T10:51:36.734Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"d7e2d677-c8bf-4ba3-8f8a-6f366d9c8004","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:36.745Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2e58a506-0992-4cff-9af8-636e2d5303a1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:36.746Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2e58a506-0992-4cff-9af8-636e2d5303a1","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:36.746Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2e58a506-0992-4cff-9af8-636e2d5303a1","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:36.746Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2e58a506-0992-4cff-9af8-636e2d5303a1"}
{"level":"error","ts":"2026-07-10T10:51:36.751Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2e58a506-0992-4cff-9af8-636e2d5303a1","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:36.751Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2e58a506-0992-4cff-9af8-636e2d5303a1"}
{"level":"error","ts":"2026-07-10T10:51:36.751Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"2e58a506-0992-4cff-9af8-636e2d5303a1","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:38.706Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6dba8961-2025-45d9-934c-1eeebe9d3863","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:38.706Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6dba8961-2025-45d9-934c-1eeebe9d3863","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:38.706Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6dba8961-2025-45d9-934c-1eeebe9d3863","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:38.707Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6dba8961-2025-45d9-934c-1eeebe9d3863"}
{"level":"error","ts":"2026-07-10T10:51:38.712Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6dba8961-2025-45d9-934c-1eeebe9d3863","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:38.712Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6dba8961-2025-45d9-934c-1eeebe9d3863"}
{"level":"error","ts":"2026-07-10T10:51:38.712Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"6dba8961-2025-45d9-934c-1eeebe9d3863","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:43.023Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"847a62f4-c9d5-464d-9aae-11f1285b7248","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:43.023Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"847a62f4-c9d5-464d-9aae-11f1285b7248","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:43.023Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"847a62f4-c9d5-464d-9aae-11f1285b7248","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:43.023Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"847a62f4-c9d5-464d-9aae-11f1285b7248"}
{"level":"error","ts":"2026-07-10T10:51:43.029Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"847a62f4-c9d5-464d-9aae-11f1285b7248","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:43.029Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"847a62f4-c9d5-464d-9aae-11f1285b7248"}
{"level":"error","ts":"2026-07-10T10:51:43.029Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"847a62f4-c9d5-464d-9aae-11f1285b7248","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:44.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7706ccbb-9369-4a5a-b68b-7e2a2902b3b4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:44.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7706ccbb-9369-4a5a-b68b-7e2a2902b3b4","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:44.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7706ccbb-9369-4a5a-b68b-7e2a2902b3b4","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:44.022Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7706ccbb-9369-4a5a-b68b-7e2a2902b3b4"}
{"level":"error","ts":"2026-07-10T10:51:44.028Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7706ccbb-9369-4a5a-b68b-7e2a2902b3b4","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:44.028Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7706ccbb-9369-4a5a-b68b-7e2a2902b3b4"}
{"level":"error","ts":"2026-07-10T10:51:44.028Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7706ccbb-9369-4a5a-b68b-7e2a2902b3b4","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:48.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"96eb31ee-e8c2-4604-a7b6-2dec79e6df8c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:48.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"96eb31ee-e8c2-4604-a7b6-2dec79e6df8c","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:48.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"96eb31ee-e8c2-4604-a7b6-2dec79e6df8c","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:48.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"96eb31ee-e8c2-4604-a7b6-2dec79e6df8c"}
{"level":"error","ts":"2026-07-10T10:51:48.787Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"96eb31ee-e8c2-4604-a7b6-2dec79e6df8c","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:48.787Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"96eb31ee-e8c2-4604-a7b6-2dec79e6df8c"}
{"level":"error","ts":"2026-07-10T10:51:48.787Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"96eb31ee-e8c2-4604-a7b6-2dec79e6df8c","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:49.787Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a2e42d9b-77b5-4ee4-a69a-240daa913312","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:49.787Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a2e42d9b-77b5-4ee4-a69a-240daa913312","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:49.787Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a2e42d9b-77b5-4ee4-a69a-240daa913312","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:49.787Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a2e42d9b-77b5-4ee4-a69a-240daa913312"}
{"level":"error","ts":"2026-07-10T10:51:49.793Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a2e42d9b-77b5-4ee4-a69a-240daa913312","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:49.793Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a2e42d9b-77b5-4ee4-a69a-240daa913312"}
{"level":"error","ts":"2026-07-10T10:51:49.793Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"a2e42d9b-77b5-4ee4-a69a-240daa913312","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:49.802Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7ce1c485-9508-4411-bf4e-8427e08c45c5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:51:49.802Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7ce1c485-9508-4411-bf4e-8427e08c45c5","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:51:49.802Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7ce1c485-9508-4411-bf4e-8427e08c45c5","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:51:49.802Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7ce1c485-9508-4411-bf4e-8427e08c45c5"}
{"level":"error","ts":"2026-07-10T10:51:49.807Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7ce1c485-9508-4411-bf4e-8427e08c45c5","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:51:49.807Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7ce1c485-9508-4411-bf4e-8427e08c45c5"}
{"level":"error","ts":"2026-07-10T10:51:49.807Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7ce1c485-9508-4411-bf4e-8427e08c45c5","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:07.025Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0297eb4c-311b-4f67-95db-da04f8cf4e3f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:07.025Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0297eb4c-311b-4f67-95db-da04f8cf4e3f","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:07.025Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0297eb4c-311b-4f67-95db-da04f8cf4e3f","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:07.025Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0297eb4c-311b-4f67-95db-da04f8cf4e3f"}
{"level":"error","ts":"2026-07-10T10:52:07.035Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0297eb4c-311b-4f67-95db-da04f8cf4e3f","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:07.035Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0297eb4c-311b-4f67-95db-da04f8cf4e3f"}
{"level":"error","ts":"2026-07-10T10:52:07.035Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0297eb4c-311b-4f67-95db-da04f8cf4e3f","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:08.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4693be47-b915-4da1-acfa-0a5f92b87233","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:08.024Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4693be47-b915-4da1-acfa-0a5f92b87233","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:08.024Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4693be47-b915-4da1-acfa-0a5f92b87233","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:08.024Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4693be47-b915-4da1-acfa-0a5f92b87233"}
{"level":"error","ts":"2026-07-10T10:52:08.030Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4693be47-b915-4da1-acfa-0a5f92b87233","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:08.030Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4693be47-b915-4da1-acfa-0a5f92b87233"}
{"level":"error","ts":"2026-07-10T10:52:08.030Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"4693be47-b915-4da1-acfa-0a5f92b87233","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:08.236Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"e0611ec7-3d55-4b58-8804-2e7126d3a4be","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:08.237Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"e0611ec7-3d55-4b58-8804-2e7126d3a4be","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:52:08.237Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"e0611ec7-3d55-4b58-8804-2e7126d3a4be","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:52:08.237Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"e0611ec7-3d55-4b58-8804-2e7126d3a4be","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.132.0.32","10.133.0.44"]}
{"level":"info","ts":"2026-07-10T10:52:10.431Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5ad1d47b-2e8b-4296-a30d-f534f9969e24","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:52:10.431Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5ad1d47b-2e8b-4296-a30d-f534f9969e24","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:52:10.431Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5ad1d47b-2e8b-4296-a30d-f534f9969e24","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:52:10.431Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"5ad1d47b-2e8b-4296-a30d-f534f9969e24","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:52:12.898Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"009eb0a3-f6d7-465d-bde8-712396330f11","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:12.898Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"009eb0a3-f6d7-465d-bde8-712396330f11","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:12.898Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"009eb0a3-f6d7-465d-bde8-712396330f11","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:12.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"009eb0a3-f6d7-465d-bde8-712396330f11"}
{"level":"error","ts":"2026-07-10T10:52:12.904Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"009eb0a3-f6d7-465d-bde8-712396330f11","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:12.904Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"009eb0a3-f6d7-465d-bde8-712396330f11"}
{"level":"error","ts":"2026-07-10T10:52:12.904Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"009eb0a3-f6d7-465d-bde8-712396330f11","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:13.907Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8e2b05f6-ab22-49c1-99be-d884681cdaac","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:13.907Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8e2b05f6-ab22-49c1-99be-d884681cdaac","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:13.907Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8e2b05f6-ab22-49c1-99be-d884681cdaac","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:13.907Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8e2b05f6-ab22-49c1-99be-d884681cdaac"}
{"level":"error","ts":"2026-07-10T10:52:13.913Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8e2b05f6-ab22-49c1-99be-d884681cdaac","JobId":"fail-k8s-job-26l4b","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:13.913Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8e2b05f6-ab22-49c1-99be-d884681cdaac"}
{"level":"error","ts":"2026-07-10T10:52:13.913Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"8e2b05f6-ab22-49c1-99be-d884681cdaac","error":"Job fail-k8s-job-26l4b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:13.924Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c8ada8d-31d1-4762-aaba-3aacf308e0ec","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:13.924Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c8ada8d-31d1-4762-aaba-3aacf308e0ec","Submitter K8s Job":"fail-k8s-job","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-07-10T10:52:13.924Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c8ada8d-31d1-4762-aaba-3aacf308e0ec","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-26l4b","rayClusterName":"fail-k8s-job-gpvmn","dashboardURL":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:49:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-26l4b","rayClusterName":"fail-k8s-job-gpvmn","dashboardURL":"fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-07-10T10:49:38Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:52:13.924Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"7c8ada8d-31d1-4762-aaba-3aacf308e0ec","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-10T10:52:13.937Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b433b244-8580-413f-ae8d-a52b0f0abd2c","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:13.937Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b433b244-8580-413f-ae8d-a52b0f0abd2c","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:52:13 +0000 UTC","Now":"2026-07-10T10:52:13.937Z","ShutdownTime":"2026-07-10T10:52:13.000Z"}
{"level":"info","ts":"2026-07-10T10:52:13.937Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b433b244-8580-413f-ae8d-a52b0f0abd2c","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:52:13.948Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b433b244-8580-413f-ae8d-a52b0f0abd2c","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:13.948Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b433b244-8580-413f-ae8d-a52b0f0abd2c","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:52:13.948Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"b433b244-8580-413f-ae8d-a52b0f0abd2c","RayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"96babe8d-20fd-4428-8490-9d8dca7a0a14","rayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"96babe8d-20fd-4428-8490-9d8dca7a0a14","pod":"fail-k8s-job-gpvmn-small-group-worker-lplvs","ip":"10.133.0.44"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"96babe8d-20fd-4428-8490-9d8dca7a0a14","pod":"fail-k8s-job-gpvmn-head-h5vrf","ip":"10.132.0.32"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"d41badaf-fdfd-4955-915a-800ee82b9d03"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"reconcileID":"67aa7be1-a4d4-40f4-8bc2-0993a2d79b13"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"96babe8d-20fd-4428-8490-9d8dca7a0a14","rayCluster":"fail-k8s-job-gpvmn","podIPs":["10.133.0.44","10.132.0.32"]}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"67614f9c-a9a2-4a04-b835-52556d579958","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"67614f9c-a9a2-4a04-b835-52556d579958","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:52:13 +0000 UTC","Now":"2026-07-10T10:52:13.949Z","ShutdownTime":"2026-07-10T10:52:13.000Z"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"67614f9c-a9a2-4a04-b835-52556d579958","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"67614f9c-a9a2-4a04-b835-52556d579958","RayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"67614f9c-a9a2-4a04-b835-52556d579958","isClusterDeleted":false}
{"level":"info","ts":"2026-07-10T10:52:13.949Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"67614f9c-a9a2-4a04-b835-52556d579958","RayCluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:13.957Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:13.978Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","name":"test-ns-z4qpg-fail-k8s-job-gpvmn","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:52:13.991Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","configMap":"kube-rbac-proxy-config-fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.010Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","serviceAccount":"fail-k8s-job-gpvmn-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:52:14.036Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.036Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"f9609780-e05a-40ab-a601-0a1e4ea5731c","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb","cluster":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ea998d02-9cbb-4924-9cef-ae8d1188d7ce","namespace":"test-ns-z4qpg","clusterName":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"03bee97e-0a60-4882-bcd3-bf89d163feae"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"33defb5a-4b39-48da-8004-7fb1fb445c87","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"33defb5a-4b39-48da-8004-7fb1fb445c87","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:52:13 +0000 UTC","Now":"2026-07-10T10:52:14.046Z","ShutdownTime":"2026-07-10T10:52:13.000Z"}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"33defb5a-4b39-48da-8004-7fb1fb445c87","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"33defb5a-4b39-48da-8004-7fb1fb445c87","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"33defb5a-4b39-48da-8004-7fb1fb445c87","isClusterDeleted":true}
{"level":"info","ts":"2026-07-10T10:52:14.046Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"33defb5a-4b39-48da-8004-7fb1fb445c87","RayCluster":"fail-k8s-job-gpvmn"}
{"level":"error","ts":"2026-07-10T10:52:14.050Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb","error":"rayclusters.ray.io \"fail-k8s-job-gpvmn\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:14.050Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb"}
{"level":"error","ts":"2026-07-10T10:52:14.050Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9d97a57b-9fd7-4867-8644-8ebc1caf1dfb","error":"rayclusters.ray.io \"fail-k8s-job-gpvmn\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:14.050Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"df0897db-c4f6-471a-8f52-e90ebbc60686","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:14.050Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"df0897db-c4f6-471a-8f52-e90ebbc60686"}
{"level":"info","ts":"2026-07-10T10:52:14.050Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"df0897db-c4f6-471a-8f52-e90ebbc60686","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:52:14.050Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"df0897db-c4f6-471a-8f52-e90ebbc60686","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:14.053Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"1695b290-944a-4017-877c-d0356addf7d6","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:14.053Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"1695b290-944a-4017-877c-d0356addf7d6"}
{"level":"info","ts":"2026-07-10T10:52:14.053Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"1695b290-944a-4017-877c-d0356addf7d6","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:52:14.053Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"1695b290-944a-4017-877c-d0356addf7d6","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:14.055Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"694620d9-ecf2-4d3d-b7f1-5591b46eb3b4","namespacedName":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:14.055Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"694620d9-ecf2-4d3d-b7f1-5591b46eb3b4"}
{"level":"info","ts":"2026-07-10T10:52:14.055Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"694620d9-ecf2-4d3d-b7f1-5591b46eb3b4","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:52:14.055Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"694620d9-ecf2-4d3d-b7f1-5591b46eb3b4","namespace":"test-ns-z4qpg"}
{"level":"error","ts":"2026-07-10T10:52:14.057Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ea998d02-9cbb-4924-9cef-ae8d1188d7ce","issuer":"ray-selfsigned-issuer-fail-k8s-job-gpvmn","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-gpvmn\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:14.068Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"9dffd31d-6422-4397-9e5a-93943203beb3"}
{"level":"error","ts":"2026-07-10T10:52:14.125Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ea998d02-9cbb-4924-9cef-ae8d1188d7ce","certificate":"ray-ca-certificate-fail-k8s-job-gpvmn","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-k8s-job-gpvmn\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:14.125Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"390c0046-7b67-4be3-966c-9c958f71782f"}
{"level":"info","ts":"2026-07-10T10:52:14.141Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ea998d02-9cbb-4924-9cef-ae8d1188d7ce","errorCount":2}
{"level":"info","ts":"2026-07-10T10:52:14.141Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ea998d02-9cbb-4924-9cef-ae8d1188d7ce"}
{"level":"error","ts":"2026-07-10T10:52:14.141Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"ea998d02-9cbb-4924-9cef-ae8d1188d7ce","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-gpvmn\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:14.146Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"7bb3c66b-91ca-45db-b052-c6dcf93a47a1","namespace":"test-ns-z4qpg","clusterName":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.146Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-gpvmn","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"fail-k8s-job-gpvmn","reconcileID":"7bb3c66b-91ca-45db-b052-c6dcf93a47a1","namespace":"test-ns-z4qpg","clusterName":"fail-k8s-job-gpvmn"}
{"level":"info","ts":"2026-07-10T10:52:14.944Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"892c0c8c-e79b-42f0-beb5-c336652674a2","DeletionTimestamp":"2026-07-10 10:52:14 +0000 UTC"}
{"level":"error","ts":"2026-07-10T10:52:14.944Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"892c0c8c-e79b-42f0-beb5-c336652674a2","error":"RayCluster.ray.io \"fail-k8s-job-gpvmn\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:14.944Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"892c0c8c-e79b-42f0-beb5-c336652674a2","rayJob":"fail-k8s-job-26l4b"}
{"level":"error","ts":"2026-07-10T10:52:16.945Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"892c0c8c-e79b-42f0-beb5-c336652674a2","error":"Post \"http://fail-k8s-job-gpvmn-head-svc.test-ns-z4qpg.svc.cluster.local:8265/api/jobs/fail-k8s-job-26l4b/stop\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:16.945Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"892c0c8c-e79b-42f0-beb5-c336652674a2","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:52:16.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"47ff421c-ed0f-49ac-8465-d45cd3e762ae","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:16.959Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"47ff421c-ed0f-49ac-8465-d45cd3e762ae","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:52:16.970Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"47ff421c-ed0f-49ac-8465-d45cd3e762ae"}
{"level":"info","ts":"2026-07-10T10:52:16.970Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"47ff421c-ed0f-49ac-8465-d45cd3e762ae","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:52:16.970Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"47ff421c-ed0f-49ac-8465-d45cd3e762ae","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:52:16.971Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"47ff421c-ed0f-49ac-8465-d45cd3e762ae","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:52:16.983Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"12afc19f-4a61-4535-8956-c47fd06c20cc"}
{"level":"info","ts":"2026-07-10T10:52:16.983Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:16.983Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af"}
{"level":"info","ts":"2026-07-10T10:52:16.983Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:52:16.983Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-p8tnb","rayClusterName":"stop-dh4h8","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:52:16.983Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:52:16.989Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:52:16.989Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af"}
{"level":"error","ts":"2026-07-10T10:52:16.989Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f28748e3-8e4d-451e-adaa-3e9b54ef15af","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:16.990Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3cc2ed21-5e55-4ac1-8700-34faa6eb1ab6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:16.990Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3cc2ed21-5e55-4ac1-8700-34faa6eb1ab6","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:16.990Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3cc2ed21-5e55-4ac1-8700-34faa6eb1ab6","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:16.990Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3cc2ed21-5e55-4ac1-8700-34faa6eb1ab6","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:16.994Z","logger":"raycluster-resource","msg":"default","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:16.994Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-8gpbm","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3cc2ed21-5e55-4ac1-8700-34faa6eb1ab6","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","mode":"IntegratedOAuth","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3cc2ed21-5e55-4ac1-8700-34faa6eb1ab6"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3cc2ed21-5e55-4ac1-8700-34faa6eb1ab6","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"652e33d3-ed37-4c92-bdf7-c6538fdf5986","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"87932beb-a0bc-427b-940e-972bf570110c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"87932beb-a0bc-427b-940e-972bf570110c","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"87932beb-a0bc-427b-940e-972bf570110c","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"87932beb-a0bc-427b-940e-972bf570110c"}
{"level":"info","ts":"2026-07-10T10:52:17.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"87932beb-a0bc-427b-940e-972bf570110c","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","name":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","name":"stop-8gpbm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"raycluster-resource","msg":"default","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.025Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-8gpbm","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:17.035Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"652e33d3-ed37-4c92-bdf7-c6538fdf5986","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.035Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.039Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ca4b33fe-ce90-44fb-9f5c-efa9586177e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:17.040Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","name":"stop-8gpbm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"stop-8gpbm"}}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ca4b33fe-ce90-44fb-9f5c-efa9586177e8","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"fa80e809-c02b-416c-9faf-b00ccfac3430","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ca4b33fe-ce90-44fb-9f5c-efa9586177e8","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.046Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","name":"stop-8gpbm-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619"}
{"level":"error","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"50c897f0-08f6-45af-87cd-2ba3de233619","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ca4b33fe-ce90-44fb-9f5c-efa9586177e8"}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ca4b33fe-ce90-44fb-9f5c-efa9586177e8","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:52:17.125Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","name":"stop-8gpbm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:17.131Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg","operation":"created"}
{"level":"info","ts":"2026-07-10T10:52:17.137Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:52:17.137Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd"}
{"level":"error","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"33eef9c4-0645-45f1-9972-65760cc3aacd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","name":"stop-8gpbm","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"1e6f5beb-13bc-4bfe-b7ab-280d5904a051","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"1e6f5beb-13bc-4bfe-b7ab-280d5904a051","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.138Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"1e6f5beb-13bc-4bfe-b7ab-280d5904a051","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.139Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"1e6f5beb-13bc-4bfe-b7ab-280d5904a051"}
{"level":"info","ts":"2026-07-10T10:52:17.139Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"1e6f5beb-13bc-4bfe-b7ab-280d5904a051","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:17.140Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","name":"test-ns-z4qpg-stop-8gpbm","operation":"created"}
{"level":"info","ts":"2026-07-10T10:52:17.141Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"fa80e809-c02b-416c-9faf-b00ccfac3430","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:17.144Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","name":"stop-8gpbm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"stop-8gpbm"}}
{"level":"info","ts":"2026-07-10T10:52:17.144Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1"}
{"level":"error","ts":"2026-07-10T10:52:17.144Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"80984c55-23c3-47f6-808a-6defa5ecf4b1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.145Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62"}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62"}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62"}
{"level":"error","ts":"2026-07-10T10:52:17.148Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e02d600-ffcb-4630-8bb2-67d020e34f62","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.148Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","name":"kube-rbac-proxy-config-stop-8gpbm","operation":"created"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472"}
{"level":"error","ts":"2026-07-10T10:52:17.225Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0a3ccc97-ac70-4abe-aafc-3b2582dee472","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"fa80e809-c02b-416c-9faf-b00ccfac3430","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"fa80e809-c02b-416c-9faf-b00ccfac3430","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:52:17.225Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b2b56725-f57c-4f5c-bd3c-0be48f488249","cluster":"stop-8gpbm"}
{"level":"error","ts":"2026-07-10T10:52:17.234Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.234Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1"}
{"level":"error","ts":"2026-07-10T10:52:17.234Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"82a2247c-96e6-4191-8a84-04ec0a476fe1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.235Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"d39e4bf1-597e-43b7-9dbd-a811508e32f8","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.235Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"d39e4bf1-597e-43b7-9dbd-a811508e32f8","mode":"IntegratedOAuth","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.235Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"d39e4bf1-597e-43b7-9dbd-a811508e32f8","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.235Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6","rayCluster":"stop-8gpbm","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:52:17.235Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8f07c87a-e7a3-478a-acb2-ce48969f3cc6"}
{"level":"info","ts":"2026-07-10T10:52:17.235Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"daaf0951-ad80-43b8-ad84-8f7789d02176","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.235Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"daaf0951-ad80-43b8-ad84-8f7789d02176"}
{"level":"info","ts":"2026-07-10T10:52:17.239Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b2b56725-f57c-4f5c-bd3c-0be48f488249","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"d39e4bf1-597e-43b7-9dbd-a811508e32f8","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f348a711-b6d2-43b2-91f4-4272faed65bb","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f348a711-b6d2-43b2-91f4-4272faed65bb","mode":"IntegratedOAuth","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f348a711-b6d2-43b2-91f4-4272faed65bb","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"926a6d6a-e197-4995-91df-93e58b0dec05","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"926a6d6a-e197-4995-91df-93e58b0dec05"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"51cbf131-ac8a-4ffe-85a8-290d65aa39b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"51cbf131-ac8a-4ffe-85a8-290d65aa39b6","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"51cbf131-ac8a-4ffe-85a8-290d65aa39b6","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"51cbf131-ac8a-4ffe-85a8-290d65aa39b6"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"51cbf131-ac8a-4ffe-85a8-290d65aa39b6","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b2b56725-f57c-4f5c-bd3c-0be48f488249","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b2b56725-f57c-4f5c-bd3c-0be48f488249","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257"}
{"level":"error","ts":"2026-07-10T10:52:17.325Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"20e805a4-bf88-428d-b118-e9f633a7a257","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"985d66d3-63b5-4a32-854c-a48fcbfb76fa","cluster":"stop-8gpbm"}
{"level":"error","ts":"2026-07-10T10:52:17.325Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f348a711-b6d2-43b2-91f4-4272faed65bb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.325Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f348a711-b6d2-43b2-91f4-4272faed65bb"}
{"level":"error","ts":"2026-07-10T10:52:17.325Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f348a711-b6d2-43b2-91f4-4272faed65bb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.330Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"c16886a3-daea-41dd-8532-08f79d984bac","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:17.330Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"c16886a3-daea-41dd-8532-08f79d984bac","mode":"IntegratedOAuth","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.330Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"c16886a3-daea-41dd-8532-08f79d984bac","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.338Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"985d66d3-63b5-4a32-854c-a48fcbfb76fa","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:17.339Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"c16886a3-daea-41dd-8532-08f79d984bac","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.347Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"985d66d3-63b5-4a32-854c-a48fcbfb76fa","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:17.347Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"985d66d3-63b5-4a32-854c-a48fcbfb76fa","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56"}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56"}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56"}
{"level":"error","ts":"2026-07-10T10:52:17.486Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2d4f8092-0f88-4d84-81eb-f1be1be04d56","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8"}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8"}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8"}
{"level":"error","ts":"2026-07-10T10:52:17.806Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"dc4e7268-ebaf-4754-ad25-5ab9345a56a8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439"}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439"}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","error":"head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"error","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found"}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"stop-8gpbm","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439"}
{"level":"error","ts":"2026-07-10T10:52:18.447Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e44afe4a-5f7b-4c15-bcf6-689b9fd3c439","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-8gpbm not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","mode":"IntegratedOAuth","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","cluster":"stop-8gpbm","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","cluster":"stop-8gpbm","authType":"IntegratedOAuth","serviceAccount":"stop-8gpbm-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-8gpbm","ray.io/group":"headgroup","ray.io/identifier":"stop-8gpbm-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:52:19.727Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:52:19.728Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:52:19.728Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:52:19.728Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","enabled":true}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","name":"stop-8gpbm-head-ztjq2"}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b"}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","nodeType":"worker","rayStartParams":{"address":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","rayStartCmd":"ray start  --address=stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:52:19.744Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","enabled":true}
{"level":"info","ts":"2026-07-10T10:52:19.758Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","name":"stop-8gpbm-small-group-worker-9bj6g"}
{"level":"info","ts":"2026-07-10T10:52:19.758Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.758Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.758Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-8gpbm-head-svc"},"newHead":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"}}
{"level":"info","ts":"2026-07-10T10:52:19.758Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b8d99389-d20c-405c-99e7-8fd95092c30b","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-8gpbm-head-svc"},"newHead":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"}}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5ceafbcd-9862-4d25-9b02-f4287f2719b7","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"59ca9fb6-7f24-4ad4-831b-c20c93bc0689","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"26eebf74-71ae-4c82-97f5-5d272b2dda9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"26eebf74-71ae-4c82-97f5-5d272b2dda9a","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"26eebf74-71ae-4c82-97f5-5d272b2dda9a","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:19.771Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"59ca9fb6-7f24-4ad4-831b-c20c93bc0689","rayCluster":"stop-8gpbm","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:52:19.772Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"26eebf74-71ae-4c82-97f5-5d272b2dda9a"}
{"level":"info","ts":"2026-07-10T10:52:19.772Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"26eebf74-71ae-4c82-97f5-5d272b2dda9a","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:19.779Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","name":"stop-8gpbm","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"stop-8gpbm"}}
{"level":"info","ts":"2026-07-10T10:52:19.779Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab"}
{"level":"error","ts":"2026-07-10T10:52:19.779Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e27580e3-e3df-4c69-8a7f-e3af69bc2aab","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-8gpbm\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad"}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad"}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:19.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.783Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5ceafbcd-9862-4d25-9b02-f4287f2719b7","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:19.825Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.825Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.825Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5ceafbcd-9862-4d25-9b02-f4287f2719b7","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:19.825Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5ceafbcd-9862-4d25-9b02-f4287f2719b7","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.825Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:52:19.825Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1a635af6-094a-405f-b7c1-134f40062bad","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"9db9cb12-1ac5-4bc4-abfa-b2d7cf7beddd","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"af4a5d8f-6c13-4324-91d1-bc105f40ff26","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"af4a5d8f-6c13-4324-91d1-bc105f40ff26","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"af4a5d8f-6c13-4324-91d1-bc105f40ff26","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"ba89d73f-4b51-4790-9dc4-1e6ad4d6d193","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:19.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"af4a5d8f-6c13-4324-91d1-bc105f40ff26"}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"af4a5d8f-6c13-4324-91d1-bc105f40ff26","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"ba89d73f-4b51-4790-9dc4-1e6ad4d6d193","rayCluster":"stop-8gpbm","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:19.839Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"7541f4d3-3b49-407d-b4c9-11621fb27b1e","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:19.849Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"9db9cb12-1ac5-4bc4-abfa-b2d7cf7beddd","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:19.859Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"9db9cb12-1ac5-4bc4-abfa-b2d7cf7beddd","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:19.859Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"9db9cb12-1ac5-4bc4-abfa-b2d7cf7beddd","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:19.960Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-z4qpg"},"reconcileID":"0e861933-9ac2-4a96-b001-4e8eb876a2ab"}
{"level":"info","ts":"2026-07-10T10:52:20.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"28513bfd-a274-4574-bf0b-8839fad5a574","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:20.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"28513bfd-a274-4574-bf0b-8839fad5a574","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:20.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"28513bfd-a274-4574-bf0b-8839fad5a574","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:20.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"28513bfd-a274-4574-bf0b-8839fad5a574"}
{"level":"info","ts":"2026-07-10T10:52:20.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"28513bfd-a274-4574-bf0b-8839fad5a574","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:20.214Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"1d4b93a1-e07a-4d9b-afb4-215d8d5d46de","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:20.217Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a654aabd-1634-4e20-8f0e-e590929ca466","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:20.940Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cf4f61a6-2589-43bc-ac78-4c5b609ce4dc","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"newHead":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"}}
{"level":"info","ts":"2026-07-10T10:52:21.091Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"be09f9e8-7a6e-424d-9fa3-0d5b3cb42d98","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:21.103Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"72eb2acf-256c-4f57-9c30-5411cee483b8","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:21.103Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f9526b64-5814-4f4f-8ab6-d757c9a43137","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:21.103Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a1295ca3-52fe-45b0-a3a5-2f60b745fd48","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:21.103Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a1295ca3-52fe-45b0-a3a5-2f60b745fd48","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:21.103Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a1295ca3-52fe-45b0-a3a5-2f60b745fd48","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:21.103Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"72eb2acf-256c-4f57-9c30-5411cee483b8","pod":"stop-8gpbm-head-ztjq2","ip":"10.132.0.33"}
{"level":"info","ts":"2026-07-10T10:52:21.103Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"72eb2acf-256c-4f57-9c30-5411cee483b8","pod":"stop-8gpbm-small-group-worker-9bj6g","ip":"10.133.0.48"}
{"level":"info","ts":"2026-07-10T10:52:21.104Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a1295ca3-52fe-45b0-a3a5-2f60b745fd48"}
{"level":"info","ts":"2026-07-10T10:52:21.104Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a1295ca3-52fe-45b0-a3a5-2f60b745fd48","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:21.116Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f9526b64-5814-4f4f-8ab6-d757c9a43137","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:21.117Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"72eb2acf-256c-4f57-9c30-5411cee483b8","certificate":"ray-head-cert-stop-8gpbm","dnsNames":["stop-8gpbm-head-svc","localhost","stop-8gpbm-head-svc.test-ns-z4qpg.svc","stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.33","10.133.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:52:21.128Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f9526b64-5814-4f4f-8ab6-d757c9a43137","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:21.128Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"f9526b64-5814-4f4f-8ab6-d757c9a43137","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:21.129Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"72eb2acf-256c-4f57-9c30-5411cee483b8","certificate":"ray-worker-cert-stop-8gpbm","dnsNames":["stop-8gpbm-worker-svc","localhost","stop-8gpbm-small-group","stop-8gpbm-worker-svc.test-ns-z4qpg.svc","stop-8gpbm-worker-svc.test-ns-z4qpg.svc.cluster.local","*.stop-8gpbm-worker-svc.test-ns-z4qpg.svc","*.stop-8gpbm-worker-svc.test-ns-z4qpg.svc.cluster.local","*-worker-*.test-ns-z4qpg.svc","*-worker-*.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.33","10.133.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:52:21.129Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"72eb2acf-256c-4f57-9c30-5411cee483b8","rayCluster":"stop-8gpbm","podIPs":["10.132.0.33","10.133.0.48"]}
{"level":"info","ts":"2026-07-10T10:52:21.839Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450"}
{"level":"info","ts":"2026-07-10T10:52:21.839Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450"}
{"level":"info","ts":"2026-07-10T10:52:21.839Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:21.840Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:21.841Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:21.841Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:21.841Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"3e849d84-74e0-4ca4-abe3-2a1c37931450","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:23.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c4f6ff2d-10dc-45c3-bf0f-e20711df6f18","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:23.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c4f6ff2d-10dc-45c3-bf0f-e20711df6f18","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:23.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c4f6ff2d-10dc-45c3-bf0f-e20711df6f18","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:23.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c4f6ff2d-10dc-45c3-bf0f-e20711df6f18"}
{"level":"info","ts":"2026-07-10T10:52:23.009Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c4f6ff2d-10dc-45c3-bf0f-e20711df6f18","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed"}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed"}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:25.959Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:25.960Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","podName":"stop-8gpbm-head-ztjq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:25.960Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:25.960Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"82ceb6e7-3d86-4533-8f75-1fc329a6ffed","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:26.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4963161a-6c96-4fbf-ac64-1bb09cae62ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:26.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4963161a-6c96-4fbf-ac64-1bb09cae62ce","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:26.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4963161a-6c96-4fbf-ac64-1bb09cae62ce","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:26.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4963161a-6c96-4fbf-ac64-1bb09cae62ce"}
{"level":"info","ts":"2026-07-10T10:52:26.009Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4963161a-6c96-4fbf-ac64-1bb09cae62ce","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:26.114Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"fb8b5a93-edd0-4820-b850-59be007854e5","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:26.966Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"d8a7119d-f21e-42a0-b594-9d1765e8914b","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8"}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8"}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:27.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:27.123Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:52:27.123Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"a5405eeb-c439-42c7-aab1-057e5d4ff8c8","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"04249d10-4973-4059-805d-10fbcd22767c","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"866df7dc-1c38-4024-bf76-853716fac2f9","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"04249d10-4973-4059-805d-10fbcd22767c","pod":"stop-8gpbm-head-ztjq2","ip":"10.132.0.33"}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"04249d10-4973-4059-805d-10fbcd22767c","pod":"stop-8gpbm-small-group-worker-9bj6g","ip":"10.133.0.48"}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"80ebff20-aab2-48ed-aa06-b5a3272271c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"80ebff20-aab2-48ed-aa06-b5a3272271c8","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"04249d10-4973-4059-805d-10fbcd22767c","rayCluster":"stop-8gpbm","podIPs":["10.132.0.33","10.133.0.48"]}
{"level":"info","ts":"2026-07-10T10:52:27.135Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"80ebff20-aab2-48ed-aa06-b5a3272271c8","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:27.136Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"80ebff20-aab2-48ed-aa06-b5a3272271c8"}
{"level":"info","ts":"2026-07-10T10:52:27.136Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"80ebff20-aab2-48ed-aa06-b5a3272271c8","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:27.151Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"866df7dc-1c38-4024-bf76-853716fac2f9","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:27.163Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"866df7dc-1c38-4024-bf76-853716fac2f9","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:27.163Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"866df7dc-1c38-4024-bf76-853716fac2f9","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:29.010Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ab199b12-04fe-4dec-b755-7dcf1036e19c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:29.010Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ab199b12-04fe-4dec-b755-7dcf1036e19c","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:29.010Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ab199b12-04fe-4dec-b755-7dcf1036e19c","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:29.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ab199b12-04fe-4dec-b755-7dcf1036e19c"}
{"level":"info","ts":"2026-07-10T10:52:29.010Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ab199b12-04fe-4dec-b755-7dcf1036e19c","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:29.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48"}
{"level":"info","ts":"2026-07-10T10:52:29.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48"}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:29.137Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"cee327c0-0f64-4e21-bfe1-197d313f8e48","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:32.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"96ab5496-a518-4bea-a7d3-b35d719dc15d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:32.011Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"96ab5496-a518-4bea-a7d3-b35d719dc15d","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:32.011Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"96ab5496-a518-4bea-a7d3-b35d719dc15d","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:32.011Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"96ab5496-a518-4bea-a7d3-b35d719dc15d"}
{"level":"info","ts":"2026-07-10T10:52:32.011Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"96ab5496-a518-4bea-a7d3-b35d719dc15d","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:35.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"931c2f80-b1fd-47f4-8bb8-e0a4fc30fe79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:35.011Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"931c2f80-b1fd-47f4-8bb8-e0a4fc30fe79","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:35.011Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"931c2f80-b1fd-47f4-8bb8-e0a4fc30fe79","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:35.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"931c2f80-b1fd-47f4-8bb8-e0a4fc30fe79"}
{"level":"info","ts":"2026-07-10T10:52:35.012Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"931c2f80-b1fd-47f4-8bb8-e0a4fc30fe79","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6"}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6"}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:36.011Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"2b5e087c-c81b-45c2-8b12-361e352be1d6","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef"}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef"}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","podName":"stop-8gpbm-head-ztjq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:52:37.017Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"b46b0501-915a-4c2b-93a0-f736ae5d43ef","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"22a02e0f-4a5b-454b-9115-53f35c07e314","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"631786f1-384a-4a8e-8232-823e431e3a6d","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2b31ab68-41d2-4dcf-90f3-6dd845a4f0d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2b31ab68-41d2-4dcf-90f3-6dd845a4f0d8","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2b31ab68-41d2-4dcf-90f3-6dd845a4f0d8","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"631786f1-384a-4a8e-8232-823e431e3a6d","pod":"stop-8gpbm-head-ztjq2","ip":"10.132.0.33"}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"631786f1-384a-4a8e-8232-823e431e3a6d","pod":"stop-8gpbm-small-group-worker-9bj6g","ip":"10.133.0.48"}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"631786f1-384a-4a8e-8232-823e431e3a6d","rayCluster":"stop-8gpbm","podIPs":["10.132.0.33","10.133.0.48"]}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2b31ab68-41d2-4dcf-90f3-6dd845a4f0d8"}
{"level":"info","ts":"2026-07-10T10:52:37.029Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2b31ab68-41d2-4dcf-90f3-6dd845a4f0d8","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:37.039Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"22a02e0f-4a5b-454b-9115-53f35c07e314","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:37.049Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"22a02e0f-4a5b-454b-9115-53f35c07e314","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:37.049Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"22a02e0f-4a5b-454b-9115-53f35c07e314","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:38.013Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f42d9f24-7695-4a1e-8911-3cb79ea02e3e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:38.013Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f42d9f24-7695-4a1e-8911-3cb79ea02e3e","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:38.013Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f42d9f24-7695-4a1e-8911-3cb79ea02e3e","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:38.013Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f42d9f24-7695-4a1e-8911-3cb79ea02e3e"}
{"level":"info","ts":"2026-07-10T10:52:38.013Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f42d9f24-7695-4a1e-8911-3cb79ea02e3e","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a"}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a"}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:52:38.153Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"e8411252-63c0-4fdd-9a75-c2cd13d2293a","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"03b4f7ee-a5c5-4059-b758-5348dd6e6a57","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8debf8c8-b8f9-4332-a7b6-e9c6e2ed92fe","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"698b5eb0-679e-4383-b018-ae010dd0ebba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"698b5eb0-679e-4383-b018-ae010dd0ebba","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"698b5eb0-679e-4383-b018-ae010dd0ebba","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8debf8c8-b8f9-4332-a7b6-e9c6e2ed92fe","pod":"stop-8gpbm-head-ztjq2","ip":"10.132.0.33"}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8debf8c8-b8f9-4332-a7b6-e9c6e2ed92fe","pod":"stop-8gpbm-small-group-worker-9bj6g","ip":"10.133.0.48"}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8debf8c8-b8f9-4332-a7b6-e9c6e2ed92fe","rayCluster":"stop-8gpbm","podIPs":["10.132.0.33","10.133.0.48"]}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"698b5eb0-679e-4383-b018-ae010dd0ebba"}
{"level":"info","ts":"2026-07-10T10:52:38.166Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"698b5eb0-679e-4383-b018-ae010dd0ebba","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:38.176Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"03b4f7ee-a5c5-4059-b758-5348dd6e6a57","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:38.188Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"03b4f7ee-a5c5-4059-b758-5348dd6e6a57","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:38.188Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"03b4f7ee-a5c5-4059-b758-5348dd6e6a57","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:39.029Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16"}
{"level":"info","ts":"2026-07-10T10:52:39.029Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16"}
{"level":"info","ts":"2026-07-10T10:52:39.029Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:39.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:39.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:39.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:39.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:39.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:39.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:39.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:39.030Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:52:39.030Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","podName":"stop-8gpbm-small-group-worker-9bj6g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:52:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:52:39.030Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:39.030Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"0530322f-708b-41e3-b423-19e8ee055b16","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:41.013Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"121ef30a-a50b-40fb-b9b2-c981fe6a0403","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:41.013Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"121ef30a-a50b-40fb-b9b2-c981fe6a0403","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:41.013Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"121ef30a-a50b-40fb-b9b2-c981fe6a0403","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:41.013Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"121ef30a-a50b-40fb-b9b2-c981fe6a0403"}
{"level":"info","ts":"2026-07-10T10:52:41.013Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"121ef30a-a50b-40fb-b9b2-c981fe6a0403","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:44.014Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3312176e-fdb9-422e-9a0a-5373925736fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:44.014Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3312176e-fdb9-422e-9a0a-5373925736fc","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:44.014Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3312176e-fdb9-422e-9a0a-5373925736fc","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:44.014Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3312176e-fdb9-422e-9a0a-5373925736fc"}
{"level":"info","ts":"2026-07-10T10:52:44.014Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3312176e-fdb9-422e-9a0a-5373925736fc","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:47.015Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"67514a29-8f6b-4a47-9d1d-54921c93c32b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:47.015Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"67514a29-8f6b-4a47-9d1d-54921c93c32b","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:47.015Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"67514a29-8f6b-4a47-9d1d-54921c93c32b","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:47.015Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"67514a29-8f6b-4a47-9d1d-54921c93c32b"}
{"level":"info","ts":"2026-07-10T10:52:47.015Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"67514a29-8f6b-4a47-9d1d-54921c93c32b","RayCluster":"stop-8gpbm","State":""}
{"level":"info","ts":"2026-07-10T10:52:47.236Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"e889c711-49d0-4437-96f9-b84a2165e51b","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:47.236Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"e889c711-49d0-4437-96f9-b84a2165e51b","pod":"stop-8gpbm-head-ztjq2","ip":"10.132.0.33"}
{"level":"info","ts":"2026-07-10T10:52:47.236Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"e889c711-49d0-4437-96f9-b84a2165e51b","pod":"stop-8gpbm-small-group-worker-9bj6g","ip":"10.133.0.48"}
{"level":"info","ts":"2026-07-10T10:52:47.236Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"e889c711-49d0-4437-96f9-b84a2165e51b","rayCluster":"stop-8gpbm","podIPs":["10.132.0.33","10.133.0.48"]}
{"level":"info","ts":"2026-07-10T10:52:48.060Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad"}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad"}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:52:48.061Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"54302a90-cfa9-41b1-806a-093477af49ad","name":"stop-8gpbm","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"aa10be46-a9c4-48b7-a67d-6193f4d273f6","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b208d5e2-8636-4db5-8998-e04f76be0129","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b208d5e2-8636-4db5-8998-e04f76be0129","pod":"stop-8gpbm-head-ztjq2","ip":"10.132.0.33"}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b208d5e2-8636-4db5-8998-e04f76be0129","pod":"stop-8gpbm-small-group-worker-9bj6g","ip":"10.133.0.48"}
{"level":"info","ts":"2026-07-10T10:52:48.074Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b208d5e2-8636-4db5-8998-e04f76be0129","rayCluster":"stop-8gpbm","podIPs":["10.132.0.33","10.133.0.48"]}
{"level":"info","ts":"2026-07-10T10:52:48.075Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28"}
{"level":"info","ts":"2026-07-10T10:52:48.075Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","head service name":"stop-8gpbm-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:48.075Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","head service URL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:52:48.075Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28"}
{"level":"info","ts":"2026-07-10T10:52:48.075Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","command":["if","!","ray","job","status","--address","http://stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","stop-4pcdb",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","--no-wait","--submission-id","stop-4pcdb","--","python /home/ray/jobs/stop.py",";","fi",";","ray","job","logs","--address","http://stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","--follow","stop-4pcdb"]}
{"level":"info","ts":"2026-07-10T10:52:48.084Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-07-10T10:52:48.084Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","SubmissionMode":"K8sJobMode","RayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:52:48.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"43205326-84c1-4f5f-813a-688bc5e03d28","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:52:48.092Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"aa10be46-a9c4-48b7-a67d-6193f4d273f6","name":"stop-8gpbm-head"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","head service name":"stop-8gpbm-head-svc","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","head service URL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","SubmissionMode":"K8sJobMode","RayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:52:48.097Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:52:48.106Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"aa10be46-a9c4-48b7-a67d-6193f4d273f6","name":"stop-8gpbm-workers"}
{"level":"info","ts":"2026-07-10T10:52:48.106Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"aa10be46-a9c4-48b7-a67d-6193f4d273f6","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.133Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:52:48.133Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326"}
{"level":"error","ts":"2026-07-10T10:52:48.133Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"11408bd2-2763-4699-89b7-70bebd3ab326","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.133Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b83139cb-9040-45b7-b1c5-19548756d513","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.133Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b83139cb-9040-45b7-b1c5-19548756d513","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.133Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b83139cb-9040-45b7-b1c5-19548756d513","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.133Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b83139cb-9040-45b7-b1c5-19548756d513"}
{"level":"error","ts":"2026-07-10T10:52:48.147Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b83139cb-9040-45b7-b1c5-19548756d513","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.147Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b83139cb-9040-45b7-b1c5-19548756d513"}
{"level":"error","ts":"2026-07-10T10:52:48.147Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b83139cb-9040-45b7-b1c5-19548756d513","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.147Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"54682072-cdb0-40c9-bd0a-4537ea1bc542","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.147Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"54682072-cdb0-40c9-bd0a-4537ea1bc542","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.147Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"54682072-cdb0-40c9-bd0a-4537ea1bc542","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.147Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"54682072-cdb0-40c9-bd0a-4537ea1bc542"}
{"level":"error","ts":"2026-07-10T10:52:48.152Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"54682072-cdb0-40c9-bd0a-4537ea1bc542","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.152Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"54682072-cdb0-40c9-bd0a-4537ea1bc542"}
{"level":"error","ts":"2026-07-10T10:52:48.152Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"54682072-cdb0-40c9-bd0a-4537ea1bc542","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.157Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"fdd85a2b-1106-4dd3-83bc-33f5ff32b079","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.157Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"fdd85a2b-1106-4dd3-83bc-33f5ff32b079","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.157Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"fdd85a2b-1106-4dd3-83bc-33f5ff32b079","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.157Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"fdd85a2b-1106-4dd3-83bc-33f5ff32b079"}
{"level":"error","ts":"2026-07-10T10:52:48.162Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"fdd85a2b-1106-4dd3-83bc-33f5ff32b079","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.162Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"fdd85a2b-1106-4dd3-83bc-33f5ff32b079"}
{"level":"error","ts":"2026-07-10T10:52:48.162Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"fdd85a2b-1106-4dd3-83bc-33f5ff32b079","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.202Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"355fd9da-64a9-47d3-8b7d-6836d0e7d6f9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.202Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"355fd9da-64a9-47d3-8b7d-6836d0e7d6f9","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.202Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"355fd9da-64a9-47d3-8b7d-6836d0e7d6f9","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.202Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"355fd9da-64a9-47d3-8b7d-6836d0e7d6f9"}
{"level":"error","ts":"2026-07-10T10:52:48.208Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"355fd9da-64a9-47d3-8b7d-6836d0e7d6f9","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.208Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"355fd9da-64a9-47d3-8b7d-6836d0e7d6f9"}
{"level":"error","ts":"2026-07-10T10:52:48.208Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"355fd9da-64a9-47d3-8b7d-6836d0e7d6f9","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"817ff3b0-57d6-454c-b05a-e1c39f43f7af","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"817ff3b0-57d6-454c-b05a-e1c39f43f7af","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"817ff3b0-57d6-454c-b05a-e1c39f43f7af","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.289Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"817ff3b0-57d6-454c-b05a-e1c39f43f7af"}
{"level":"error","ts":"2026-07-10T10:52:48.294Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"817ff3b0-57d6-454c-b05a-e1c39f43f7af","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.294Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"817ff3b0-57d6-454c-b05a-e1c39f43f7af"}
{"level":"error","ts":"2026-07-10T10:52:48.294Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"817ff3b0-57d6-454c-b05a-e1c39f43f7af","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.454Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a31aa21d-4e2d-4db8-82a8-c1511ac2cab4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.454Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a31aa21d-4e2d-4db8-82a8-c1511ac2cab4","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.455Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a31aa21d-4e2d-4db8-82a8-c1511ac2cab4","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.455Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a31aa21d-4e2d-4db8-82a8-c1511ac2cab4"}
{"level":"error","ts":"2026-07-10T10:52:48.460Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a31aa21d-4e2d-4db8-82a8-c1511ac2cab4","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.460Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a31aa21d-4e2d-4db8-82a8-c1511ac2cab4"}
{"level":"error","ts":"2026-07-10T10:52:48.460Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a31aa21d-4e2d-4db8-82a8-c1511ac2cab4","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"bf2afba7-a671-4f25-9c3c-862b4381d64b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:48.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"bf2afba7-a671-4f25-9c3c-862b4381d64b","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:48.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"bf2afba7-a671-4f25-9c3c-862b4381d64b","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:48.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"bf2afba7-a671-4f25-9c3c-862b4381d64b"}
{"level":"error","ts":"2026-07-10T10:52:48.786Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"bf2afba7-a671-4f25-9c3c-862b4381d64b","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:48.786Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"bf2afba7-a671-4f25-9c3c-862b4381d64b"}
{"level":"error","ts":"2026-07-10T10:52:48.786Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"bf2afba7-a671-4f25-9c3c-862b4381d64b","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:49.108Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4276a2b9-722b-4f00-abaa-5e7247a2e601","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:49.108Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4276a2b9-722b-4f00-abaa-5e7247a2e601","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:49.108Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4276a2b9-722b-4f00-abaa-5e7247a2e601","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:49.108Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4276a2b9-722b-4f00-abaa-5e7247a2e601"}
{"level":"error","ts":"2026-07-10T10:52:49.114Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4276a2b9-722b-4f00-abaa-5e7247a2e601","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:49.114Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4276a2b9-722b-4f00-abaa-5e7247a2e601"}
{"level":"error","ts":"2026-07-10T10:52:49.114Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"4276a2b9-722b-4f00-abaa-5e7247a2e601","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:49.427Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"09d3a7ba-b786-43d1-aa5b-a317efda0911","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:49.428Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"09d3a7ba-b786-43d1-aa5b-a317efda0911","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:49.428Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"09d3a7ba-b786-43d1-aa5b-a317efda0911","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:49.428Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"09d3a7ba-b786-43d1-aa5b-a317efda0911"}
{"level":"error","ts":"2026-07-10T10:52:49.434Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"09d3a7ba-b786-43d1-aa5b-a317efda0911","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:49.434Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"09d3a7ba-b786-43d1-aa5b-a317efda0911"}
{"level":"error","ts":"2026-07-10T10:52:49.434Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"09d3a7ba-b786-43d1-aa5b-a317efda0911","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40"}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40"}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","1 head service found":"stop-8gpbm-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","Found 1 head Pod":"stop-8gpbm-head-ztjq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","head Pod":"stop-8gpbm-head-ztjq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-8gpbm-head-ztjq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","worker Pod":"stop-8gpbm-small-group-worker-9bj6g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-8gpbm-small-group-worker-9bj6g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:50.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:50.076Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:50.076Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"516dc964-a551-465b-8b45-c8c6dbb3ee40","seconds":300}
{"level":"info","ts":"2026-07-10T10:52:51.995Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"075e14df-52aa-47e3-ade4-0bcd914103fb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:51.995Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"075e14df-52aa-47e3-ade4-0bcd914103fb","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:51.995Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"075e14df-52aa-47e3-ade4-0bcd914103fb","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:51.995Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"075e14df-52aa-47e3-ade4-0bcd914103fb"}
{"level":"error","ts":"2026-07-10T10:52:52.001Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"075e14df-52aa-47e3-ade4-0bcd914103fb","JobId":"stop-4pcdb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:52.001Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"075e14df-52aa-47e3-ade4-0bcd914103fb"}
{"level":"error","ts":"2026-07-10T10:52:52.001Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"075e14df-52aa-47e3-ade4-0bcd914103fb","error":"Job stop-4pcdb does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:52:57.121Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a132b6f7-3485-4b33-8c74-505cc8f7b477","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:57.122Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a132b6f7-3485-4b33-8c74-505cc8f7b477","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:57.122Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a132b6f7-3485-4b33-8c74-505cc8f7b477","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:57.122Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a132b6f7-3485-4b33-8c74-505cc8f7b477"}
{"level":"info","ts":"2026-07-10T10:52:57.128Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a132b6f7-3485-4b33-8c74-505cc8f7b477","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:52:57.128Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"a132b6f7-3485-4b33-8c74-505cc8f7b477","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:52:57.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3811659d-5a1e-4fbd-abf5-380c97db98fc","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:52:57.142Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3811659d-5a1e-4fbd-abf5-380c97db98fc","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:52:57.142Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3811659d-5a1e-4fbd-abf5-380c97db98fc","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:52:57.142Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3811659d-5a1e-4fbd-abf5-380c97db98fc"}
{"level":"info","ts":"2026-07-10T10:52:57.145Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3811659d-5a1e-4fbd-abf5-380c97db98fc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103"}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103"}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:52:59.927Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"951e0eeb-77c4-438d-afb5-075995b18103","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:00.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"37910d58-9938-45fc-a0bf-6ed671774721","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:00.142Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"37910d58-9938-45fc-a0bf-6ed671774721","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:00.142Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"37910d58-9938-45fc-a0bf-6ed671774721","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:00.142Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"37910d58-9938-45fc-a0bf-6ed671774721"}
{"level":"info","ts":"2026-07-10T10:53:00.146Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"37910d58-9938-45fc-a0bf-6ed671774721","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:00.146Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"37910d58-9938-45fc-a0bf-6ed671774721","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:00.160Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"7bc7306b-ecb3-490c-ae65-a18ffdf53174","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:00.160Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"7bc7306b-ecb3-490c-ae65-a18ffdf53174","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:00.160Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"7bc7306b-ecb3-490c-ae65-a18ffdf53174","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:00.160Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"7bc7306b-ecb3-490c-ae65-a18ffdf53174"}
{"level":"info","ts":"2026-07-10T10:53:00.163Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"7bc7306b-ecb3-490c-ae65-a18ffdf53174","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:03.161Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"6b0f8294-5cf8-4743-a858-01be46dbf1ea","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:03.161Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"6b0f8294-5cf8-4743-a858-01be46dbf1ea","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:03.161Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"6b0f8294-5cf8-4743-a858-01be46dbf1ea","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:03.162Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"6b0f8294-5cf8-4743-a858-01be46dbf1ea"}
{"level":"info","ts":"2026-07-10T10:53:03.165Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"6b0f8294-5cf8-4743-a858-01be46dbf1ea","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:06.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"5e035439-7b91-407c-be08-9596528599a8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:06.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"5e035439-7b91-407c-be08-9596528599a8","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:06.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"5e035439-7b91-407c-be08-9596528599a8","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:06.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"5e035439-7b91-407c-be08-9596528599a8"}
{"level":"info","ts":"2026-07-10T10:53:06.169Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"5e035439-7b91-407c-be08-9596528599a8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:09.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3751e34f-a2f2-47b5-99eb-018445538da1","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:09.170Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3751e34f-a2f2-47b5-99eb-018445538da1","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:09.170Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3751e34f-a2f2-47b5-99eb-018445538da1","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:09.170Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3751e34f-a2f2-47b5-99eb-018445538da1"}
{"level":"info","ts":"2026-07-10T10:53:09.173Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"3751e34f-a2f2-47b5-99eb-018445538da1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:10.432Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0c2cf0f5-7627-4430-964f-72b1d48c2cc4","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:10.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0c2cf0f5-7627-4430-964f-72b1d48c2cc4","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:53:10.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0c2cf0f5-7627-4430-964f-72b1d48c2cc4","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:53:10.433Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0c2cf0f5-7627-4430-964f-72b1d48c2cc4","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:53:12.175Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"23df2311-0cdb-415f-a23c-c48725fba0fe","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:12.175Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"23df2311-0cdb-415f-a23c-c48725fba0fe","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:12.175Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"23df2311-0cdb-415f-a23c-c48725fba0fe","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:12.175Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"23df2311-0cdb-415f-a23c-c48725fba0fe"}
{"level":"info","ts":"2026-07-10T10:53:12.178Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"23df2311-0cdb-415f-a23c-c48725fba0fe","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:15.179Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f44316da-f3c7-4d6f-9edc-2ae13444a0e1","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:15.179Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f44316da-f3c7-4d6f-9edc-2ae13444a0e1","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:15.179Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f44316da-f3c7-4d6f-9edc-2ae13444a0e1","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:15.179Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f44316da-f3c7-4d6f-9edc-2ae13444a0e1"}
{"level":"info","ts":"2026-07-10T10:53:15.183Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"f44316da-f3c7-4d6f-9edc-2ae13444a0e1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:18.184Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ee1012f5-5d57-4d60-aebd-64e528b81885","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:18.184Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ee1012f5-5d57-4d60-aebd-64e528b81885","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:18.184Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ee1012f5-5d57-4d60-aebd-64e528b81885","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:18.184Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ee1012f5-5d57-4d60-aebd-64e528b81885"}
{"level":"info","ts":"2026-07-10T10:53:18.187Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"ee1012f5-5d57-4d60-aebd-64e528b81885","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:21.188Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c5d5ad26-5c30-48bc-a299-f19ec45b379b","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:21.188Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c5d5ad26-5c30-48bc-a299-f19ec45b379b","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:21.188Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c5d5ad26-5c30-48bc-a299-f19ec45b379b","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:21.188Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c5d5ad26-5c30-48bc-a299-f19ec45b379b"}
{"level":"info","ts":"2026-07-10T10:53:21.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c5d5ad26-5c30-48bc-a299-f19ec45b379b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:21.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"c5d5ad26-5c30-48bc-a299-f19ec45b379b","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:21.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b403bc18-17d3-4d43-9606-96777204abdb","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:21.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b403bc18-17d3-4d43-9606-96777204abdb","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:21.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b403bc18-17d3-4d43-9606-96777204abdb","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:21.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b403bc18-17d3-4d43-9606-96777204abdb"}
{"level":"info","ts":"2026-07-10T10:53:21.208Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"b403bc18-17d3-4d43-9606-96777204abdb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:23.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"dae8b1a6-6189-4bf4-b879-0c112b7a9b66","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:23.239Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"dae8b1a6-6189-4bf4-b879-0c112b7a9b66","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:23.239Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"dae8b1a6-6189-4bf4-b879-0c112b7a9b66","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:23.239Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"dae8b1a6-6189-4bf4-b879-0c112b7a9b66"}
{"level":"info","ts":"2026-07-10T10:53:23.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"dae8b1a6-6189-4bf4-b879-0c112b7a9b66","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:24.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"aaa23dc5-ac47-47ac-8e1f-e2dac938ae17","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:24.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"aaa23dc5-ac47-47ac-8e1f-e2dac938ae17","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"aaa23dc5-ac47-47ac-8e1f-e2dac938ae17","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"aaa23dc5-ac47-47ac-8e1f-e2dac938ae17"}
{"level":"info","ts":"2026-07-10T10:53:24.208Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"aaa23dc5-ac47-47ac-8e1f-e2dac938ae17","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:24.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2e2b4f8e-b6e7-49b5-a7cf-2b278450ee05","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:24.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2e2b4f8e-b6e7-49b5-a7cf-2b278450ee05","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2e2b4f8e-b6e7-49b5-a7cf-2b278450ee05","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.247Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2e2b4f8e-b6e7-49b5-a7cf-2b278450ee05"}
{"level":"info","ts":"2026-07-10T10:53:24.250Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"2e2b4f8e-b6e7-49b5-a7cf-2b278450ee05","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:24.262Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"70720d5b-2b3a-4d83-9179-ab18e597823b","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:24.262Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"70720d5b-2b3a-4d83-9179-ab18e597823b","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.262Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"70720d5b-2b3a-4d83-9179-ab18e597823b","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.262Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"70720d5b-2b3a-4d83-9179-ab18e597823b"}
{"level":"info","ts":"2026-07-10T10:53:24.265Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"70720d5b-2b3a-4d83-9179-ab18e597823b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:52:55Z","endTime":"2026-07-10T10:53:18Z"},"jobId":"stop-4pcdb","rayClusterName":"stop-8gpbm","dashboardURL":"stop-8gpbm-head-svc.test-ns-z4qpg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-07-10T10:52:16Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:52:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:52:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-8gpbm-head-ztjq2","serviceName":"stop-8gpbm-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:52:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:52:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:52:17Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:52:17Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:24.265Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"70720d5b-2b3a-4d83-9179-ab18e597823b","old JobStatus":"STOPPED","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-10T10:53:24.279Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"00a7067a-b97b-46b2-add9-734e8aa12ff2","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:24.279Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"00a7067a-b97b-46b2-add9-734e8aa12ff2","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:53:24 +0000 UTC","Now":"2026-07-10T10:53:24.279Z","ShutdownTime":"2026-07-10T10:53:24.000Z"}
{"level":"info","ts":"2026-07-10T10:53:24.402Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"852fd5d7-9c79-469a-b5a2-8766083d8a1f","DeletionTimestamp":"2026-07-10 10:53:24 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:53:24.402Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"852fd5d7-9c79-469a-b5a2-8766083d8a1f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-07-10T10:53:24.416Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"c336d203-f7f3-4e35-a5fd-19d116311fc2","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.416Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"c336d203-f7f3-4e35-a5fd-19d116311fc2"}
{"level":"error","ts":"2026-07-10T10:53:24.416Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"c336d203-f7f3-4e35-a5fd-19d116311fc2","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.416Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"39b55f31-0b20-4d22-9f36-51c169ef0dc3"}
{"level":"error","ts":"2026-07-10T10:53:24.422Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"bada239c-b6df-459f-8c7b-3985703aa539","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.422Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"bada239c-b6df-459f-8c7b-3985703aa539"}
{"level":"error","ts":"2026-07-10T10:53:24.422Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"bada239c-b6df-459f-8c7b-3985703aa539","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.431Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"055e69ee-e104-4805-af9c-92f0272664f5"}
{"level":"error","ts":"2026-07-10T10:53:24.433Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"4b2b86d4-cc24-4317-a134-87b487b570f2","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.433Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"4b2b86d4-cc24-4317-a134-87b487b570f2"}
{"level":"error","ts":"2026-07-10T10:53:24.433Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"4b2b86d4-cc24-4317-a134-87b487b570f2","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"2d727953-0c13-4319-a04e-70869f2298c9","rayCluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"749c514f-0e9f-40c2-b2c7-4b12c145d8eb"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"reconcileID":"f0224bd8-c42c-4971-b3c4-a8f9bcd1bfb8"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"405a555d-0083-475c-8daa-a9e7a0ac626a"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"2d727953-0c13-4319-a04e-70869f2298c9","pod":"stop-8gpbm-head-ztjq2","ip":"10.132.0.33"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"2d727953-0c13-4319-a04e-70869f2298c9","pod":"stop-8gpbm-small-group-worker-9bj6g","ip":"10.133.0.48"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:53:24.438Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"2d727953-0c13-4319-a04e-70869f2298c9","rayCluster":"stop-8gpbm","podIPs":["10.132.0.33","10.133.0.48"]}
{"level":"info","ts":"2026-07-10T10:53:24.444Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:53:24.449Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","name":"test-ns-z4qpg-stop-8gpbm","namespace":"opendatahub"}
{"level":"error","ts":"2026-07-10T10:53:24.454Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"22ce0def-cbb5-45e2-b19c-a5367e832349","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.454Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"22ce0def-cbb5-45e2-b19c-a5367e832349"}
{"level":"info","ts":"2026-07-10T10:53:24.454Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","configMap":"kube-rbac-proxy-config-stop-8gpbm"}
{"level":"error","ts":"2026-07-10T10:53:24.454Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"22ce0def-cbb5-45e2-b19c-a5367e832349","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.458Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","serviceAccount":"stop-8gpbm-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:24.464Z","logger":"raycluster-resource","msg":"default","name":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.464Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-8gpbm","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"5061d65e-1d64-4f6f-bd39-cfa2a9ebeaa3","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f","cluster":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"c30b95aa-2d77-4736-b2f5-2abeb46e44ab"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7921c8ac-4ddb-4e4f-8355-82fce00fbb72","namespace":"test-ns-z4qpg","clusterName":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.473Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"cb55d47e-4524-45c8-9164-b8245f1a21bd"}
{"level":"error","ts":"2026-07-10T10:53:24.477Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f","error":"rayclusters.ray.io \"stop-8gpbm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.478Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f"}
{"level":"error","ts":"2026-07-10T10:53:24.478Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"552a7fb7-0d7e-4879-8040-a4c61e25362f","error":"rayclusters.ray.io \"stop-8gpbm\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.478Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"679bf614-527b-4358-a76d-75437d6aa41b","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.478Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"679bf614-527b-4358-a76d-75437d6aa41b"}
{"level":"info","ts":"2026-07-10T10:53:24.478Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"679bf614-527b-4358-a76d-75437d6aa41b","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:53:24.478Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"679bf614-527b-4358-a76d-75437d6aa41b","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7317e491-a294-4be9-aa81-2c3f7d7ab865","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7317e491-a294-4be9-aa81-2c3f7d7ab865"}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7317e491-a294-4be9-aa81-2c3f7d7ab865","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"error","ts":"2026-07-10T10:53:24.525Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"fa21b034-909f-4047-8fae-557436cf0055","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7317e491-a294-4be9-aa81-2c3f7d7ab865","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"fa21b034-909f-4047-8fae-557436cf0055"}
{"level":"error","ts":"2026-07-10T10:53:24.525Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"fa21b034-909f-4047-8fae-557436cf0055","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"b955dc8a-7475-4743-9dec-ccb41b5d135f"}
{"level":"error","ts":"2026-07-10T10:53:24.525Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7921c8ac-4ddb-4e4f-8355-82fce00fbb72","issuer":"ray-selfsigned-issuer-stop-8gpbm","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-8gpbm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"1c27bcfb-1a9c-4d9b-8f59-ca63512a6e30","namespacedName":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"1c27bcfb-1a9c-4d9b-8f59-ca63512a6e30"}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"1c27bcfb-1a9c-4d9b-8f59-ca63512a6e30","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:53:24.525Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"1c27bcfb-1a9c-4d9b-8f59-ca63512a6e30","namespace":"test-ns-z4qpg"}
{"level":"error","ts":"2026-07-10T10:53:24.528Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7921c8ac-4ddb-4e4f-8355-82fce00fbb72","certificate":"ray-head-cert-stop-8gpbm","error":"certificates.cert-manager.io \"ray-head-cert-stop-8gpbm\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.546Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7921c8ac-4ddb-4e4f-8355-82fce00fbb72","errorCount":2}
{"level":"info","ts":"2026-07-10T10:53:24.547Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7921c8ac-4ddb-4e4f-8355-82fce00fbb72"}
{"level":"error","ts":"2026-07-10T10:53:24.547Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"7921c8ac-4ddb-4e4f-8355-82fce00fbb72","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-8gpbm\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.552Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8843f474-f334-4eff-ab4e-a0cef268cd90","namespace":"test-ns-z4qpg","clusterName":"stop-8gpbm"}
{"level":"info","ts":"2026-07-10T10:53:24.552Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-8gpbm","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"stop-8gpbm","reconcileID":"8843f474-f334-4eff-ab4e-a0cef268cd90","namespace":"test-ns-z4qpg","clusterName":"stop-8gpbm"}
{"level":"error","ts":"2026-07-10T10:53:24.605Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"284c75c9-0bfe-45c9-be36-5f6e8d9397e6","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.605Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"284c75c9-0bfe-45c9-be36-5f6e8d9397e6"}
{"level":"error","ts":"2026-07-10T10:53:24.605Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"284c75c9-0bfe-45c9-be36-5f6e8d9397e6","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:24.766Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"565d21c5-f24c-4aa9-90ac-4fe221480df9","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:24.766Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"565d21c5-f24c-4aa9-90ac-4fe221480df9"}
{"level":"error","ts":"2026-07-10T10:53:24.766Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"565d21c5-f24c-4aa9-90ac-4fe221480df9","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:25.087Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"64ce5438-c4c8-414a-890c-a66ec6ecf541","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:25.087Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"64ce5438-c4c8-414a-890c-a66ec6ecf541"}
{"level":"error","ts":"2026-07-10T10:53:25.087Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"64ce5438-c4c8-414a-890c-a66ec6ecf541","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:25.728Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"652aae27-b9fd-428e-98cf-0102011bbaba","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:25.728Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"652aae27-b9fd-428e-98cf-0102011bbaba"}
{"level":"error","ts":"2026-07-10T10:53:25.728Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"652aae27-b9fd-428e-98cf-0102011bbaba","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:27.009Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"ae5bed60-4ca0-4e69-91a7-c323dd77b491","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:27.009Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"ae5bed60-4ca0-4e69-91a7-c323dd77b491"}
{"level":"error","ts":"2026-07-10T10:53:27.009Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"ae5bed60-4ca0-4e69-91a7-c323dd77b491","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:27.209Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-z4qpg"},"reconcileID":"0e490d3d-5a4f-4cbb-ba3a-2d1ee80a370c"}
{"level":"info","ts":"2026-07-10T10:53:29.431Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"5c8c1f49-d132-4804-9741-d3f501a85220","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:29.431Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"5c8c1f49-d132-4804-9741-d3f501a85220","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:53:29.445Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"5c8c1f49-d132-4804-9741-d3f501a85220"}
{"level":"info","ts":"2026-07-10T10:53:29.445Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"5c8c1f49-d132-4804-9741-d3f501a85220","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:53:29.445Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"5c8c1f49-d132-4804-9741-d3f501a85220","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-mk9bd","rayClusterName":"long-running-nq5tx","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:29.445Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"5c8c1f49-d132-4804-9741-d3f501a85220","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:53:29.461Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"66699102-6888-43bb-9966-23c6d00a8db0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:29.461Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"66699102-6888-43bb-9966-23c6d00a8db0","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.461Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"66699102-6888-43bb-9966-23c6d00a8db0","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.461Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"66699102-6888-43bb-9966-23c6d00a8db0","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.468Z","logger":"raycluster-resource","msg":"default","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.469Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1c91c4d8-d5a3-4dd3-9c5c-fc581e13b305","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"66699102-6888-43bb-9966-23c6d00a8db0","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"66699102-6888-43bb-9966-23c6d00a8db0"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"66699102-6888-43bb-9966-23c6d00a8db0","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"518305b8-42e4-437f-bba1-4a5e5094582e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"518305b8-42e4-437f-bba1-4a5e5094582e","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"518305b8-42e4-437f-bba1-4a5e5094582e","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"518305b8-42e4-437f-bba1-4a5e5094582e"}
{"level":"info","ts":"2026-07-10T10:53:29.493Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"518305b8-42e4-437f-bba1-4a5e5094582e","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:29.505Z","logger":"raycluster-resource","msg":"default","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.506Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:53:29.508Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","name":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:29.508Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:29.508Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.508Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:29.508Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.509Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.509Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.509Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:53:29.509Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","name":"long-running-nq5tx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:29.513Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.513Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.518Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"ca2083fa-f115-4408-861a-436854315f52","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:29.518Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"ca2083fa-f115-4408-861a-436854315f52","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.518Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"ca2083fa-f115-4408-861a-436854315f52","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.525Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"ca2083fa-f115-4408-861a-436854315f52"}
{"level":"info","ts":"2026-07-10T10:53:29.525Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"ca2083fa-f115-4408-861a-436854315f52","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:29.525Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1c91c4d8-d5a3-4dd3-9c5c-fc581e13b305","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.525Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b037e403-d348-4424-acac-f1555e8a6a44","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.536Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","name":"long-running-nq5tx-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:29.537Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:53:29.537Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","name":"long-running-nq5tx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-nq5tx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"long-running-nq5tx"}}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3"}
{"level":"error","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"dd476ef3-262d-404a-a1f4-e3a20c8be0d3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45"}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45"}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.538Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:53:29.539Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","name":"long-running-nq5tx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:29.543Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b037e403-d348-4424-acac-f1555e8a6a44","name":"long-running-nq5tx-head"}
{"level":"info","ts":"2026-07-10T10:53:29.545Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","name":"kuberay-gateway-access","namespace":"test-ns-z4qpg","operation":"created"}
{"level":"info","ts":"2026-07-10T10:53:29.554Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45"}
{"level":"error","ts":"2026-07-10T10:53:29.554Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8f16e6aa-cded-4454-939a-858ae1e0cb45","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.555Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f"}
{"level":"info","ts":"2026-07-10T10:53:29.555Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f"}
{"level":"info","ts":"2026-07-10T10:53:29.557Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","name":"test-ns-z4qpg-long-running-nq5tx","operation":"created"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"55e56bc0-115e-4508-a39e-7a3f1bd439ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"55e56bc0-115e-4508-a39e-7a3f1bd439ad","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"55e56bc0-115e-4508-a39e-7a3f1bd439ad","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"55e56bc0-115e-4508-a39e-7a3f1bd439ad"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"55e56bc0-115e-4508-a39e-7a3f1bd439ad","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f"}
{"level":"error","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"3e806959-2636-4c4a-b28c-357a7be87a8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"e52349d6-d8ea-4ac6-9d36-76c82b709a88","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"e52349d6-d8ea-4ac6-9d36-76c82b709a88"}
{"level":"error","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"e52349d6-d8ea-4ac6-9d36-76c82b709a88","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a"}
{"level":"info","ts":"2026-07-10T10:53:29.625Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:53:29.630Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:29.631Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:29.631Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.725Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:29.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.725Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.725Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a"}
{"level":"error","ts":"2026-07-10T10:53:29.725Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ab1f8894-67ef-4a58-91a8-fd4d16df9c0a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:29.645Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b037e403-d348-4424-acac-f1555e8a6a44","name":"long-running-nq5tx-workers"}
{"level":"error","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b037e403-d348-4424-acac-f1555e8a6a44","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1998208a-e6d2-4ed5-8e23-f268dc65a056","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.645Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","rayCluster":"long-running-nq5tx","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:53:29.726Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-nq5tx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c"}
{"level":"error","ts":"2026-07-10T10:53:29.726Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"caa4083b-ed85-4288-8bf4-8dd3b9fafe1c","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-nq5tx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"f8479aa2-e930-4dfc-841f-10dafca233a0","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.636Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","name":"kube-rbac-proxy-config-long-running-nq5tx","operation":"created"}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5"}
{"level":"error","ts":"2026-07-10T10:53:29.726Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"9a3b1f7f-6908-4ef4-b208-ab74cacf11d5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397"}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397"}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397"}
{"level":"error","ts":"2026-07-10T10:53:29.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7ec977b3-aa19-4f57-ade9-2663fd232397","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:29.925Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"f8479aa2-e930-4dfc-841f-10dafca233a0","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-nq5tx\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:29.925Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"f8479aa2-e930-4dfc-841f-10dafca233a0","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-nq5tx\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:29.925Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-nq5tx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.925Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"f8479aa2-e930-4dfc-841f-10dafca233a0"}
{"level":"info","ts":"2026-07-10T10:53:29.925Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8"}
{"level":"error","ts":"2026-07-10T10:53:29.925Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0fe067a6-7b92-47b5-9382-852d66c009f8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-nq5tx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:29.925Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"f8479aa2-e930-4dfc-841f-10dafca233a0","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-nq5tx\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:29.925Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"20fd67f7-7c2e-4781-bfa2-fb0586d866d9","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.926Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"20fd67f7-7c2e-4781-bfa2-fb0586d866d9","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.926Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"20fd67f7-7c2e-4781-bfa2-fb0586d866d9","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.926Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"23e65ba5-c9c3-4759-ad1f-3b1ac9ce7f48","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.926Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"23e65ba5-c9c3-4759-ad1f-3b1ac9ce7f48"}
{"level":"info","ts":"2026-07-10T10:53:29.930Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1998208a-e6d2-4ed5-8e23-f268dc65a056","name":"long-running-nq5tx-head"}
{"level":"info","ts":"2026-07-10T10:53:29.935Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ea6674a9-5e7f-4a86-9ac2-2ee4635bcfbb","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.936Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ea6674a9-5e7f-4a86-9ac2-2ee4635bcfbb"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"20fd67f7-7c2e-4781-bfa2-fb0586d866d9","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"9ac198ea-bd5c-48c4-aeec-4373393bd2ac","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"9ac198ea-bd5c-48c4-aeec-4373393bd2ac","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"9ac198ea-bd5c-48c4-aeec-4373393bd2ac","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0ed9b874-ff8f-4cd6-ba53-12dc3cbb72a1","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0ed9b874-ff8f-4cd6-ba53-12dc3cbb72a1"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"3fe501f5-2b1d-4e17-99c0-c1ceeb1a72fd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"3fe501f5-2b1d-4e17-99c0-c1ceeb1a72fd","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:29.942Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"3fe501f5-2b1d-4e17-99c0-c1ceeb1a72fd","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:29.943Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"3fe501f5-2b1d-4e17-99c0-c1ceeb1a72fd"}
{"level":"info","ts":"2026-07-10T10:53:29.943Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"3fe501f5-2b1d-4e17-99c0-c1ceeb1a72fd","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027"}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027"}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:30.025Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"9ac198ea-bd5c-48c4-aeec-4373393bd2ac","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-nq5tx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"9ac198ea-bd5c-48c4-aeec-4373393bd2ac"}
{"level":"error","ts":"2026-07-10T10:53:30.025Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"9ac198ea-bd5c-48c4-aeec-4373393bd2ac","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-nq5tx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027"}
{"level":"error","ts":"2026-07-10T10:53:30.025Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b06ea881-8d9b-4282-8737-62b117003027","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:30.030Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"43984466-3447-4652-a4d1-54eadfd398bd","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:30.030Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"43984466-3447-4652-a4d1-54eadfd398bd","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:30.030Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"43984466-3447-4652-a4d1-54eadfd398bd","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:30.032Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1998208a-e6d2-4ed5-8e23-f268dc65a056","name":"long-running-nq5tx-workers"}
{"level":"info","ts":"2026-07-10T10:53:30.032Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1998208a-e6d2-4ed5-8e23-f268dc65a056","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:30.032Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"e5236155-92b2-4eb1-9fca-ac99ca42d97a","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:30.044Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"43984466-3447-4652-a4d1-54eadfd398bd","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:30.051Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"e5236155-92b2-4eb1-9fca-ac99ca42d97a","name":"long-running-nq5tx-head"}
{"level":"info","ts":"2026-07-10T10:53:30.130Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"e5236155-92b2-4eb1-9fca-ac99ca42d97a","name":"long-running-nq5tx-workers"}
{"level":"info","ts":"2026-07-10T10:53:30.130Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"e5236155-92b2-4eb1-9fca-ac99ca42d97a","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c"}
{"level":"info","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c"}
{"level":"info","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.347Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.347Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c"}
{"level":"error","ts":"2026-07-10T10:53:30.347Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6969b43f-a377-4590-965a-ff0164dbe83c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f"}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f"}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","error":"head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"error","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found"}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f"}
{"level":"error","ts":"2026-07-10T10:53:30.987Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b84cf400-307b-4733-a6d2-a9e13345616f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-nq5tx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:32.268Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","enabled":true}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","name":"long-running-nq5tx-head-nr5qg"}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064"}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","nodeType":"worker","rayStartParams":{"address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","rayStartCmd":"ray start  --address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:53:32.286Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","enabled":true}
{"level":"info","ts":"2026-07-10T10:53:32.300Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","name":"long-running-nq5tx-small-group-worker-fwlzv"}
{"level":"info","ts":"2026-07-10T10:53:32.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.300Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-nq5tx-head-svc"},"newHead":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"}}
{"level":"info","ts":"2026-07-10T10:53:32.300Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"0f44e254-e852-4482-aa09-62aed8914064","name":"long-running-nq5tx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"756d5cee-d1f0-42da-bf22-2160aa1aa71e","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"74d6a824-087a-4f30-ae35-6cb950c1d4b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"74d6a824-087a-4f30-ae35-6cb950c1d4b4","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ca17d557-ef27-4686-90a6-05d97abd7f71","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"74d6a824-087a-4f30-ae35-6cb950c1d4b4","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"74d6a824-087a-4f30-ae35-6cb950c1d4b4"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"74d6a824-087a-4f30-ae35-6cb950c1d4b4","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ca17d557-ef27-4686-90a6-05d97abd7f71","rayCluster":"long-running-nq5tx","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:32.314Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7bf613b-e6a7-465d-9e8b-1a13a69ea3d8","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:32.322Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc"}
{"level":"info","ts":"2026-07-10T10:53:32.322Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc"}
{"level":"info","ts":"2026-07-10T10:53:32.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:32.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:32.323Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"a836cac6-0d7f-4409-894e-56a96e10a5bc","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:32.331Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"756d5cee-d1f0-42da-bf22-2160aa1aa71e","name":"long-running-nq5tx-head"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.335Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.336Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:53:32.336Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"918b507b-04a0-4da4-ba28-f0ba28aedda4","name":"long-running-nq5tx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:32.343Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"756d5cee-d1f0-42da-bf22-2160aa1aa71e","name":"long-running-nq5tx-workers"}
{"level":"info","ts":"2026-07-10T10:53:32.343Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"756d5cee-d1f0-42da-bf22-2160aa1aa71e","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0790d1f8-34fc-4fde-b89b-d0d29a555c68","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"4ed0044b-a855-4482-b2fd-340397571474","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"6113770c-a569-4aba-ae42-53db3b59529d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"6113770c-a569-4aba-ae42-53db3b59529d","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.347Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","name":"long-running-nq5tx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:32.348Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"6113770c-a569-4aba-ae42-53db3b59529d","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:32.348Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"4ed0044b-a855-4482-b2fd-340397571474","rayCluster":"long-running-nq5tx","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:53:32.355Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","name":"long-running-nq5tx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-nq5tx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-z4qpg","name":"long-running-nq5tx"}}
{"level":"info","ts":"2026-07-10T10:53:32.355Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848"}
{"level":"error","ts":"2026-07-10T10:53:32.355Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c36ad232-7e93-4ece-92f3-cc99fcd0a848","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-nq5tx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:32.359Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0790d1f8-34fc-4fde-b89b-d0d29a555c68","name":"long-running-nq5tx-head"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:32.361Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"288c6281-b3c6-46dd-bafe-978a3ed279d6","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:32.369Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0790d1f8-34fc-4fde-b89b-d0d29a555c68","name":"long-running-nq5tx-workers"}
{"level":"info","ts":"2026-07-10T10:53:32.369Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"0790d1f8-34fc-4fde-b89b-d0d29a555c68","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.425Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"6113770c-a569-4aba-ae42-53db3b59529d"}
{"level":"info","ts":"2026-07-10T10:53:32.425Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"6113770c-a569-4aba-ae42-53db3b59529d","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:32.461Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fac3dc97-2b20-4ea4-80a8-0aeaae7fc7dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:32.461Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fac3dc97-2b20-4ea4-80a8-0aeaae7fc7dd","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:32.461Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fac3dc97-2b20-4ea4-80a8-0aeaae7fc7dd","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:32.462Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fac3dc97-2b20-4ea4-80a8-0aeaae7fc7dd"}
{"level":"info","ts":"2026-07-10T10:53:32.462Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fac3dc97-2b20-4ea4-80a8-0aeaae7fc7dd","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:32.741Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"d57e8ad8-9f18-4ad6-ab97-527cfaded36a","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:32.749Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"955c233b-30cf-4b75-b389-e5f74be80e99","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:33.288Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"68055824-43c2-4d7d-ac10-b7273b9b7d9a","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"newHead":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"}}
{"level":"info","ts":"2026-07-10T10:53:33.394Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"96314904-1eac-43cc-84dd-4145cc5c3af9","name":"long-running-nq5tx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"a0ee2c66-870e-4d29-8ad5-b4a249f6cc3b","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"bdb01e51-eaf4-4bda-8f77-ef53b7d12762","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"d74a953c-1c4a-41eb-b1ad-9c8a55918322","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"d74a953c-1c4a-41eb-b1ad-9c8a55918322","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"d74a953c-1c4a-41eb-b1ad-9c8a55918322","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"bdb01e51-eaf4-4bda-8f77-ef53b7d12762","pod":"long-running-nq5tx-head-nr5qg","ip":"10.132.0.34"}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"bdb01e51-eaf4-4bda-8f77-ef53b7d12762","pod":"long-running-nq5tx-small-group-worker-fwlzv","ip":"10.133.0.50"}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"d74a953c-1c4a-41eb-b1ad-9c8a55918322"}
{"level":"info","ts":"2026-07-10T10:53:33.405Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"d74a953c-1c4a-41eb-b1ad-9c8a55918322","RayCluster":"long-running-nq5tx","State":""}
{"level":"info","ts":"2026-07-10T10:53:33.416Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"a0ee2c66-870e-4d29-8ad5-b4a249f6cc3b","name":"long-running-nq5tx-head"}
{"level":"info","ts":"2026-07-10T10:53:33.417Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"bdb01e51-eaf4-4bda-8f77-ef53b7d12762","certificate":"ray-head-cert-long-running-nq5tx","dnsNames":["long-running-nq5tx-head-svc","localhost","long-running-nq5tx-head-svc.test-ns-z4qpg.svc","long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.34","10.133.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:53:33.426Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"bdb01e51-eaf4-4bda-8f77-ef53b7d12762","certificate":"ray-worker-cert-long-running-nq5tx","dnsNames":["long-running-nq5tx-worker-svc","localhost","long-running-nq5tx-small-group","long-running-nq5tx-worker-svc.test-ns-z4qpg.svc","long-running-nq5tx-worker-svc.test-ns-z4qpg.svc.cluster.local","*.long-running-nq5tx-worker-svc.test-ns-z4qpg.svc","*.long-running-nq5tx-worker-svc.test-ns-z4qpg.svc.cluster.local","*-worker-*.test-ns-z4qpg.svc","*-worker-*.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.132.0.34","10.133.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:53:33.426Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"bdb01e51-eaf4-4bda-8f77-ef53b7d12762","rayCluster":"long-running-nq5tx","podIPs":["10.132.0.34","10.133.0.50"]}
{"level":"info","ts":"2026-07-10T10:53:33.429Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"a0ee2c66-870e-4d29-8ad5-b4a249f6cc3b","name":"long-running-nq5tx-workers"}
{"level":"info","ts":"2026-07-10T10:53:33.429Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"a0ee2c66-870e-4d29-8ad5-b4a249f6cc3b","cluster":"long-running-nq5tx"}
{"level":"error","ts":"2026-07-10T10:53:34.746Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"528c77a8-949e-4e9a-a12e-838597b65155","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:34.746Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"528c77a8-949e-4e9a-a12e-838597b65155"}
{"level":"error","ts":"2026-07-10T10:53:34.746Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"528c77a8-949e-4e9a-a12e-838597b65155","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:35.406Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cebaf47f-9936-4d34-96c6-90e3ac754b0c","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:35.463Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"9dc56cf5-241d-4d4d-92ef-4feaa94dba1f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:35.463Z","logger":"controllers.RayJob","msg":"The RayJob has passed the activeDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"9dc56cf5-241d-4d4d-92ef-4feaa94dba1f","StartTime":"2026-07-10 10:53:29 +0000 UTC","ActiveDeadlineSeconds":5}
{"level":"info","ts":"2026-07-10T10:53:35.463Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"9dc56cf5-241d-4d4d-92ef-4feaa94dba1f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-mk9bd","rayClusterName":"long-running-nq5tx","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-mk9bd","rayClusterName":"long-running-nq5tx","jobDeploymentStatus":"Failed","reason":"DeadlineExceeded","message":"The RayJob has passed the activeDeadlineSeconds. StartTime: 2026-07-10 10:53:29 +0000 UTC. ActiveDeadlineSeconds: 5","startTime":"2026-07-10T10:53:29Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:35.463Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"9dc56cf5-241d-4d4d-92ef-4feaa94dba1f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-10T10:53:35.475Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"4d4da570-104e-41e3-bcc9-3691300aafad","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:35.475Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"4d4da570-104e-41e3-bcc9-3691300aafad","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-10 10:53:35 +0000 UTC","Now":"2026-07-10T10:53:35.475Z","ShutdownTime":"2026-07-10T11:03:35.000Z"}
{"level":"info","ts":"2026-07-10T10:53:35.475Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"4d4da570-104e-41e3-bcc9-3691300aafad","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:53:35.475Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"4d4da570-104e-41e3-bcc9-3691300aafad","seconds":601}
{"level":"info","ts":"2026-07-10T10:53:36.489Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-z4qpg"},"reconcileID":"dcbbffb8-f17f-4145-b012-4f22eee3b82f","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-07-10T10:53:36.522Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"managed-externally","namespace":"test-ns-z4qpg"},"reconcileID":"f01884a8-4cbf-499e-b146-01bb67874400"}
{"level":"info","ts":"2026-07-10T10:53:36.601Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-xxj65"},"reconcileID":"409ea91b-4dfd-41df-9228-5f76c92a2e4f","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:38.312Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"31acf672-4e97-4ef9-89f6-bdaaf3b005f6","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:38.414Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba"}
{"level":"info","ts":"2026-07-10T10:53:38.414Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba"}
{"level":"info","ts":"2026-07-10T10:53:38.414Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:38.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:38.415Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"786dfbbd-886e-4208-9799-29317e9ce9ba","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:38.475Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fd0256fa-4fa4-497d-93da-fd9d5d5e0b71","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:38.475Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fd0256fa-4fa4-497d-93da-fd9d5d5e0b71","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-10 10:53:35 +0000 UTC","Now":"2026-07-10T10:53:38.475Z","ShutdownTime":"2026-07-10T11:03:35.000Z"}
{"level":"info","ts":"2026-07-10T10:53:38.475Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fd0256fa-4fa4-497d-93da-fd9d5d5e0b71","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:53:38.475Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"fd0256fa-4fa4-497d-93da-fd9d5d5e0b71","seconds":598}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:39.319Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"b5f6f0c2-22f9-4c13-b6e8-6981517a6d67","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34"}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34"}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:53:39.425Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c02e103b-bb04-480e-a8d6-82e096928b34","name":"long-running-nq5tx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1a6a6c47-87a4-4a22-a0fb-712f749298fa","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"dfc85e9f-00ba-4277-b9f5-7a6fe5423a4d","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"dfc85e9f-00ba-4277-b9f5-7a6fe5423a4d","pod":"long-running-nq5tx-head-nr5qg","ip":"10.132.0.34"}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"dfc85e9f-00ba-4277-b9f5-7a6fe5423a4d","pod":"long-running-nq5tx-small-group-worker-fwlzv","ip":"10.133.0.50"}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"f66dbd17-c7ff-4c72-9294-9a03651dae04","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"f66dbd17-c7ff-4c72-9294-9a03651dae04","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-10 10:53:35 +0000 UTC","Now":"2026-07-10T10:53:39.440Z","ShutdownTime":"2026-07-10T11:03:35.000Z"}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"f66dbd17-c7ff-4c72-9294-9a03651dae04","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"f66dbd17-c7ff-4c72-9294-9a03651dae04","seconds":597}
{"level":"info","ts":"2026-07-10T10:53:39.440Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"dfc85e9f-00ba-4277-b9f5-7a6fe5423a4d","rayCluster":"long-running-nq5tx","podIPs":["10.132.0.34","10.133.0.50"]}
{"level":"info","ts":"2026-07-10T10:53:39.445Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1a6a6c47-87a4-4a22-a0fb-712f749298fa","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:39.615Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"5de9f313-9693-4bf9-938d-bc3e3b6bd1c9","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.615Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"5de9f313-9693-4bf9-938d-bc3e3b6bd1c9","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:53:39.623Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"5de9f313-9693-4bf9-938d-bc3e3b6bd1c9"}
{"level":"info","ts":"2026-07-10T10:53:39.623Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"5de9f313-9693-4bf9-938d-bc3e3b6bd1c9","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:53:39.623Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"5de9f313-9693-4bf9-938d-bc3e3b6bd1c9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5gxzv","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:39.623Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"5de9f313-9693-4bf9-938d-bc3e3b6bd1c9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:53:39.630Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.630Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.630Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:39.630Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","head service name":"raycluster-head-svc","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:53:39.630Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","head service URL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:53:39.630Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0"}
{"level":"info","ts":"2026-07-10T10:53:39.630Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","counter-5gxzv",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-5gxzv","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","--follow","counter-5gxzv"]}
{"level":"info","ts":"2026-07-10T10:53:39.638Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-10T10:53:39.638Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.638Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5gxzv","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:39.638Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"291245f9-e625-48a4-88f8-d53b79b6c7b0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","head service name":"raycluster-head-svc","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","head service URL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5gxzv","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:39.647Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:39.653Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:53:39.653Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d"}
{"level":"error","ts":"2026-07-10T10:53:39.653Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c4a5f409-2fe9-4c98-b4c6-c3992f77b73d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.653Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"32053c55-43c4-49ce-a639-ac2ab9f320ad","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.653Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"32053c55-43c4-49ce-a639-ac2ab9f320ad","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.653Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"32053c55-43c4-49ce-a639-ac2ab9f320ad","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:39.661Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"32053c55-43c4-49ce-a639-ac2ab9f320ad","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.661Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"32053c55-43c4-49ce-a639-ac2ab9f320ad"}
{"level":"error","ts":"2026-07-10T10:53:39.661Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"32053c55-43c4-49ce-a639-ac2ab9f320ad","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.661Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"6782aa85-43bf-46e7-ad1e-9d59c550d390","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.662Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"6782aa85-43bf-46e7-ad1e-9d59c550d390","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.662Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"6782aa85-43bf-46e7-ad1e-9d59c550d390","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:39.666Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"6782aa85-43bf-46e7-ad1e-9d59c550d390","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.666Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"6782aa85-43bf-46e7-ad1e-9d59c550d390"}
{"level":"error","ts":"2026-07-10T10:53:39.666Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"6782aa85-43bf-46e7-ad1e-9d59c550d390","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.672Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4815becc-e246-495b-81c2-067eed596b59","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.672Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4815becc-e246-495b-81c2-067eed596b59","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.672Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4815becc-e246-495b-81c2-067eed596b59","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:39.729Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4815becc-e246-495b-81c2-067eed596b59","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.729Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4815becc-e246-495b-81c2-067eed596b59"}
{"level":"error","ts":"2026-07-10T10:53:39.729Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4815becc-e246-495b-81c2-067eed596b59","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.770Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a0883345-de8e-4ea6-9ee6-8b31df7fa6b9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.770Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a0883345-de8e-4ea6-9ee6-8b31df7fa6b9","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.770Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a0883345-de8e-4ea6-9ee6-8b31df7fa6b9","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:39.775Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a0883345-de8e-4ea6-9ee6-8b31df7fa6b9","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.775Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a0883345-de8e-4ea6-9ee6-8b31df7fa6b9"}
{"level":"error","ts":"2026-07-10T10:53:39.775Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a0883345-de8e-4ea6-9ee6-8b31df7fa6b9","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.856Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"adc430b5-9f35-40ab-aef4-cbb87499238f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:39.856Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"adc430b5-9f35-40ab-aef4-cbb87499238f","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:39.856Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"adc430b5-9f35-40ab-aef4-cbb87499238f","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:39.861Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"adc430b5-9f35-40ab-aef4-cbb87499238f","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:39.861Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"adc430b5-9f35-40ab-aef4-cbb87499238f"}
{"level":"error","ts":"2026-07-10T10:53:39.861Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"adc430b5-9f35-40ab-aef4-cbb87499238f","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:40.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"dfae992b-0469-41e3-ab22-78562122c21b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:40.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"dfae992b-0469-41e3-ab22-78562122c21b","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:40.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"dfae992b-0469-41e3-ab22-78562122c21b","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:40.027Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"dfae992b-0469-41e3-ab22-78562122c21b","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:40.027Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"dfae992b-0469-41e3-ab22-78562122c21b"}
{"level":"error","ts":"2026-07-10T10:53:40.027Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"dfae992b-0469-41e3-ab22-78562122c21b","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:40.348Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ff078ae4-c5aa-45ea-a9af-f5736cb0629c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:40.348Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ff078ae4-c5aa-45ea-a9af-f5736cb0629c","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:40.348Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ff078ae4-c5aa-45ea-a9af-f5736cb0629c","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:40.354Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ff078ae4-c5aa-45ea-a9af-f5736cb0629c","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:40.354Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ff078ae4-c5aa-45ea-a9af-f5736cb0629c"}
{"level":"error","ts":"2026-07-10T10:53:40.354Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ff078ae4-c5aa-45ea-a9af-f5736cb0629c","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:40.661Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"fc02385a-5cfd-42fc-8415-95ce2a751e83","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:40.661Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"fc02385a-5cfd-42fc-8415-95ce2a751e83","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:40.661Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"fc02385a-5cfd-42fc-8415-95ce2a751e83","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:40.667Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"fc02385a-5cfd-42fc-8415-95ce2a751e83","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:40.667Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"fc02385a-5cfd-42fc-8415-95ce2a751e83"}
{"level":"error","ts":"2026-07-10T10:53:40.667Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"fc02385a-5cfd-42fc-8415-95ce2a751e83","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:40.994Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"180acf39-5d42-49d0-8611-a58e60881793","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:40.994Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"180acf39-5d42-49d0-8611-a58e60881793","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:40.995Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"180acf39-5d42-49d0-8611-a58e60881793","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:41.000Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"180acf39-5d42-49d0-8611-a58e60881793","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:41.000Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"180acf39-5d42-49d0-8611-a58e60881793"}
{"level":"error","ts":"2026-07-10T10:53:41.000Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"180acf39-5d42-49d0-8611-a58e60881793","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7"}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7"}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:41.440Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"18852eea-dfc2-4309-b47c-62ce5a622fb7","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd"}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd"}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:41.616Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8bb99952-cb90-4d7d-99d0-99470ef0aecd","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa"}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa"}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:41.620Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"da50d7c2-0b08-4625-8607-0db9073e1eaa","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40"}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40"}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:41.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.641Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:41.641Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"4032501e-0cf7-42fc-bb7e-4f0841769a40","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25"}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25"}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-nq5tx-head-nr5qg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","podName":"long-running-nq5tx-head-nr5qg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:53:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:41.644Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"6fc940ab-e74a-4fa6-b632-41deec620f25","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:43.560Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"690c7168-1be0-46db-8d74-a380c7279dad","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:43.560Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"690c7168-1be0-46db-8d74-a380c7279dad","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:43.560Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"690c7168-1be0-46db-8d74-a380c7279dad","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:43.565Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"690c7168-1be0-46db-8d74-a380c7279dad","JobId":"counter-5gxzv","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:43.565Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"690c7168-1be0-46db-8d74-a380c7279dad"}
{"level":"error","ts":"2026-07-10T10:53:43.565Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"690c7168-1be0-46db-8d74-a380c7279dad","error":"Job counter-5gxzv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:53:44.986Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"6e01789e-94b8-4fb8-8013-7040ee5c794f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:44.987Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"6e01789e-94b8-4fb8-8013-7040ee5c794f"}
{"level":"error","ts":"2026-07-10T10:53:44.987Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"6e01789e-94b8-4fb8-8013-7040ee5c794f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.080Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689"}
{"level":"info","ts":"2026-07-10T10:53:45.080Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689"}
{"level":"info","ts":"2026-07-10T10:53:45.080Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","Found 1 head Pod":"long-running-nq5tx-head-nr5qg","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-10 10:53:38 +0000 UTC,FinishedAt:2026-07-10 10:53:44 +0000 UTC,ContainerID:cri-o://63e467cb404732a050f7d075dae3268a0d19dc89b412e9c3de9d602d0d1dcc7a,}"}
{"level":"info","ts":"2026-07-10T10:53:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","head Pod":"long-running-nq5tx-head-nr5qg","shouldDelete":true,"reason":"The head Pod long-running-nq5tx-head-nr5qg status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T10:53:45.086Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod long-running-nq5tx-head-nr5qg status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.086Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","error":"The head Pod long-running-nq5tx-head-nr5qg status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-10T10:53:45.086Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:53:45.125Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-10T10:53:45.125Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","name":"long-running-nq5tx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:45.138Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689"}
{"level":"error","ts":"2026-07-10T10:53:45.138Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7014db54-ab3b-4994-b235-5bcd10472689","error":"The head Pod long-running-nq5tx-head-nr5qg status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"8c5f71de-e534-4e71-bd73-c4eb97552eaf","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"8c5f71de-e534-4e71-bd73-c4eb97552eaf","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-10 10:53:35 +0000 UTC","Now":"2026-07-10T10:53:45.139Z","ShutdownTime":"2026-07-10T11:03:35.000Z"}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"8c5f71de-e534-4e71-bd73-c4eb97552eaf","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"8c5f71de-e534-4e71-bd73-c4eb97552eaf","seconds":591}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"d608bac7-72d6-49e8-9c0f-e45fec7b593a","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"d608bac7-72d6-49e8-9c0f-e45fec7b593a","pod":"long-running-nq5tx-head-nr5qg","ip":"10.132.0.34"}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"d608bac7-72d6-49e8-9c0f-e45fec7b593a","pod":"long-running-nq5tx-small-group-worker-fwlzv","ip":"10.133.0.50"}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"d608bac7-72d6-49e8-9c0f-e45fec7b593a","rayCluster":"long-running-nq5tx","podIPs":["10.132.0.34","10.133.0.50"]}
{"level":"info","ts":"2026-07-10T10:53:45.139Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"523f9d17-394f-4ac3-ac2a-0a0908eeb23d","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.143Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7"}
{"level":"info","ts":"2026-07-10T10:53:45.143Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7"}
{"level":"info","ts":"2026-07-10T10:53:45.143Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:45.144Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"81860dd6-2ea1-4b45-a7a3-777b05cfa4e7","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:45.225Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"523f9d17-394f-4ac3-ac2a-0a0908eeb23d","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","worker Pod":"long-running-nq5tx-small-group-worker-fwlzv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-nq5tx-small-group-worker-fwlzv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:53:45.458Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","podName":"long-running-nq5tx-head-nr5qg","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:53:45.459Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:53:45.459Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"09f3c64a-46db-45ec-a1fa-692274c2c597","seconds":300}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.463Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"newHead":{"serviceName":"long-running-nq5tx-head-svc"}}
{"level":"info","ts":"2026-07-10T10:53:45.466Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","name":"long-running-nq5tx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"long-running-nq5tx-head-nr5qg","serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:53:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-nq5tx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:53:29Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:53:29Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:53:45Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:53:45.477Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde"}
{"level":"error","ts":"2026-07-10T10:53:45.477Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7eb63f1b-18b0-488f-8d23-1eae2a2e4dde","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.478Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"247449b8-04b0-4d02-915a-dc99ac17626c","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.478Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"61317ba7-fb85-4c8b-bf4d-badb37c1fe05","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.478Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"307c3a19-3d89-47ee-a6a6-26a893344568","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:45.478Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"307c3a19-3d89-47ee-a6a6-26a893344568","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-10 10:53:35 +0000 UTC","Now":"2026-07-10T10:53:45.478Z","ShutdownTime":"2026-07-10T11:03:35.000Z"}
{"level":"info","ts":"2026-07-10T10:53:45.478Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"307c3a19-3d89-47ee-a6a6-26a893344568","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-10T10:53:45.478Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"307c3a19-3d89-47ee-a6a6-26a893344568","seconds":591}
{"level":"info","ts":"2026-07-10T10:53:45.478Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"61317ba7-fb85-4c8b-bf4d-badb37c1fe05","pod":"long-running-nq5tx-small-group-worker-fwlzv","ip":"10.133.0.50"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"247449b8-04b0-4d02-915a-dc99ac17626c","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.483Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827"}
{"level":"error","ts":"2026-07-10T10:53:45.486Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"54fca110-5171-452e-bbe7-1b650bdb1827","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.491Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"61317ba7-fb85-4c8b-bf4d-badb37c1fe05","certificate":"ray-head-cert-long-running-nq5tx","dnsNames":["long-running-nq5tx-head-svc","localhost","long-running-nq5tx-head-svc.test-ns-z4qpg.svc","long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.133.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.497Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:45.500Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.500Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:45.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.500Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:45.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.501Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6"}
{"level":"error","ts":"2026-07-10T10:53:45.501Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"db637230-2426-4c63-ae62-38087799aaa6","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.501Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"61317ba7-fb85-4c8b-bf4d-badb37c1fe05","certificate":"ray-worker-cert-long-running-nq5tx","dnsNames":["long-running-nq5tx-worker-svc","localhost","long-running-nq5tx-small-group","long-running-nq5tx-worker-svc.test-ns-z4qpg.svc","long-running-nq5tx-worker-svc.test-ns-z4qpg.svc.cluster.local","*.long-running-nq5tx-worker-svc.test-ns-z4qpg.svc","*.long-running-nq5tx-worker-svc.test-ns-z4qpg.svc.cluster.local","*-worker-*.test-ns-z4qpg.svc","*-worker-*.test-ns-z4qpg.svc.cluster.local"],"ipAddresses":["10.133.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:53:45.501Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"61317ba7-fb85-4c8b-bf4d-badb37c1fe05","rayCluster":"long-running-nq5tx","podIPs":["10.133.0.50"]}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.521Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:45.527Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.528Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:45.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.528Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:45.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.528Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa"}
{"level":"error","ts":"2026-07-10T10:53:45.528Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"8cf051f8-2ed1-42c0-b09c-2e33ad75d6aa","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.568Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d"}
{"level":"info","ts":"2026-07-10T10:53:45.568Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d"}
{"level":"info","ts":"2026-07-10T10:53:45.568Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.568Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d"}
{"level":"info","ts":"2026-07-10T10:53:45.568Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.568Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d"}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.569Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d"}
{"level":"error","ts":"2026-07-10T10:53:45.573Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"c50b892b-9ffd-4c8b-a38d-bbb1d35d2a3d","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.653Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36"}
{"level":"info","ts":"2026-07-10T10:53:45.653Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36"}
{"level":"info","ts":"2026-07-10T10:53:45.653Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36"}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36"}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.654Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:45.657Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.657Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:45.657Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.657Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:45.658Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.658Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.658Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36"}
{"level":"error","ts":"2026-07-10T10:53:45.658Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7a6c619b-697d-4d28-b2b7-814d0cfdec36","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:45.819Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854"}
{"level":"error","ts":"2026-07-10T10:53:45.822Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"05a5ec63-2af6-4a23-ba93-7582e8c4d854","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:46.143Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527"}
{"level":"error","ts":"2026-07-10T10:53:46.147Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"28a82671-9667-49a6-8845-087e54efe527","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:46.788Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:46.789Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:46.789Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001"}
{"level":"error","ts":"2026-07-10T10:53:46.793Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cf27d327-48ac-4f4a-bbe2-8a21cc6d7001","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:48.074Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110"}
{"level":"error","ts":"2026-07-10T10:53:48.078Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"7cc677d7-aded-4e28-a977-6d75df158110","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:48.686Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a9087e3c-2001-48de-be21-5f1ffde0dd9f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:48.686Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a9087e3c-2001-48de-be21-5f1ffde0dd9f","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:48.686Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a9087e3c-2001-48de-be21-5f1ffde0dd9f","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:48.692Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a9087e3c-2001-48de-be21-5f1ffde0dd9f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:48.692Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"a9087e3c-2001-48de-be21-5f1ffde0dd9f","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:48.704Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c8ccece3-8f3a-4457-852b-eb19df628adc","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:48.704Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c8ccece3-8f3a-4457-852b-eb19df628adc","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:48.704Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c8ccece3-8f3a-4457-852b-eb19df628adc","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:48.707Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"c8ccece3-8f3a-4457-852b-eb19df628adc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:50.639Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:50.643Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:50.643Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:50.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:50.644Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:50.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:50.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:50.644Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914"}
{"level":"error","ts":"2026-07-10T10:53:50.644Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"1db080af-3b9f-486b-8408-a993eaea1914","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:51.704Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"e9516858-e861-41c7-a77d-30ed4a533314","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:51.704Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"e9516858-e861-41c7-a77d-30ed4a533314","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:51.704Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"e9516858-e861-41c7-a77d-30ed4a533314","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:51.707Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"e9516858-e861-41c7-a77d-30ed4a533314","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:51.707Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"e9516858-e861-41c7-a77d-30ed4a533314","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:51.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"2eeebb0f-1ee9-4a00-940b-e6d574e27f8e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:51.720Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"2eeebb0f-1ee9-4a00-940b-e6d574e27f8e","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:51.720Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"2eeebb0f-1ee9-4a00-940b-e6d574e27f8e","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:51.723Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"2eeebb0f-1ee9-4a00-940b-e6d574e27f8e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:52.697Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"adba2387-5851-4f0d-86c4-776272f803c6","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.697Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"adba2387-5851-4f0d-86c4-776272f803c6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:53:52.703Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"adba2387-5851-4f0d-86c4-776272f803c6"}
{"level":"info","ts":"2026-07-10T10:53:52.703Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"adba2387-5851-4f0d-86c4-776272f803c6","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-10T10:53:52.703Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"adba2387-5851-4f0d-86c4-776272f803c6","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-rb92h","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:52.703Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"adba2387-5851-4f0d-86c4-776272f803c6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-10T10:53:52.712Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.712Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.712Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:52.712Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","head service name":"raycluster-head-svc","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:53:52.712Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","head service URL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:53:52.712Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e"}
{"level":"info","ts":"2026-07-10T10:53:52.712Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","fail-rb92h",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","--no-wait","--submission-id","fail-rb92h","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","--follow","fail-rb92h"]}
{"level":"info","ts":"2026-07-10T10:53:52.720Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-07-10T10:53:52.721Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.721Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-rb92h","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:52.721Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b1577cda-c4a4-46c9-8d3f-132d389d3d9e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","head service name":"raycluster-head-svc","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","head service URL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-rb92h","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-10T10:53:52.728Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:53:52.733Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-10T10:53:52.733Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba"}
{"level":"error","ts":"2026-07-10T10:53:52.733Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c243df9f-2dce-48ea-be33-e8bfec9a71ba","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f804c1c2-4cbe-4a15-a55b-38e29c125f9e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.733Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f804c1c2-4cbe-4a15-a55b-38e29c125f9e","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.733Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f804c1c2-4cbe-4a15-a55b-38e29c125f9e","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:52.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f804c1c2-4cbe-4a15-a55b-38e29c125f9e","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.736Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f804c1c2-4cbe-4a15-a55b-38e29c125f9e"}
{"level":"error","ts":"2026-07-10T10:53:52.736Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f804c1c2-4cbe-4a15-a55b-38e29c125f9e","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a5934284-0114-413f-aad3-57f363e83948","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.739Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a5934284-0114-413f-aad3-57f363e83948","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.739Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a5934284-0114-413f-aad3-57f363e83948","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:52.743Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a5934284-0114-413f-aad3-57f363e83948","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.743Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a5934284-0114-413f-aad3-57f363e83948"}
{"level":"error","ts":"2026-07-10T10:53:52.743Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a5934284-0114-413f-aad3-57f363e83948","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.744Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"1e6c6945-d350-43ea-8e4d-f6eed0c7e072","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.744Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"1e6c6945-d350-43ea-8e4d-f6eed0c7e072","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.744Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"1e6c6945-d350-43ea-8e4d-f6eed0c7e072","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:52.748Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"1e6c6945-d350-43ea-8e4d-f6eed0c7e072","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.748Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"1e6c6945-d350-43ea-8e4d-f6eed0c7e072"}
{"level":"error","ts":"2026-07-10T10:53:52.748Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"1e6c6945-d350-43ea-8e4d-f6eed0c7e072","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.764Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"6a77d8e7-7fbc-4c20-a873-791c0637e9a2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.764Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"6a77d8e7-7fbc-4c20-a873-791c0637e9a2","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.764Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"6a77d8e7-7fbc-4c20-a873-791c0637e9a2","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:52.769Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"6a77d8e7-7fbc-4c20-a873-791c0637e9a2","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.769Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"6a77d8e7-7fbc-4c20-a873-791c0637e9a2"}
{"level":"error","ts":"2026-07-10T10:53:52.769Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"6a77d8e7-7fbc-4c20-a873-791c0637e9a2","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"17d3b4bf-2303-4275-af8d-4f23e288164e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:52.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"17d3b4bf-2303-4275-af8d-4f23e288164e","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:52.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"17d3b4bf-2303-4275-af8d-4f23e288164e","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:52.854Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"17d3b4bf-2303-4275-af8d-4f23e288164e","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:52.854Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"17d3b4bf-2303-4275-af8d-4f23e288164e"}
{"level":"error","ts":"2026-07-10T10:53:52.854Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"17d3b4bf-2303-4275-af8d-4f23e288164e","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.015Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"0fb7aed4-0ec0-407d-b710-58ed199005f8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:53.015Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"0fb7aed4-0ec0-407d-b710-58ed199005f8","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:53.015Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"0fb7aed4-0ec0-407d-b710-58ed199005f8","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:53.020Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"0fb7aed4-0ec0-407d-b710-58ed199005f8","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.020Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"0fb7aed4-0ec0-407d-b710-58ed199005f8"}
{"level":"error","ts":"2026-07-10T10:53:53.020Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"0fb7aed4-0ec0-407d-b710-58ed199005f8","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.340Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"67316523-be23-478b-a0d6-2f8195fb344c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:53.340Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"67316523-be23-478b-a0d6-2f8195fb344c","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:53.340Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"67316523-be23-478b-a0d6-2f8195fb344c","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:53.345Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"67316523-be23-478b-a0d6-2f8195fb344c","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.345Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"67316523-be23-478b-a0d6-2f8195fb344c"}
{"level":"error","ts":"2026-07-10T10:53:53.345Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"67316523-be23-478b-a0d6-2f8195fb344c","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"5c655620-21f3-4a90-a28b-9e6598ab0b7f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:53.739Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"5c655620-21f3-4a90-a28b-9e6598ab0b7f","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:53.739Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"5c655620-21f3-4a90-a28b-9e6598ab0b7f","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:53.744Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"5c655620-21f3-4a90-a28b-9e6598ab0b7f","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.744Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"5c655620-21f3-4a90-a28b-9e6598ab0b7f"}
{"level":"error","ts":"2026-07-10T10:53:53.744Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"5c655620-21f3-4a90-a28b-9e6598ab0b7f","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.986Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"48fd4e4f-a92c-4940-a1ce-2c0b3bab1ba5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:53.986Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"48fd4e4f-a92c-4940-a1ce-2c0b3bab1ba5","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:53.986Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"48fd4e4f-a92c-4940-a1ce-2c0b3bab1ba5","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:53.991Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"48fd4e4f-a92c-4940-a1ce-2c0b3bab1ba5","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:53.991Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"48fd4e4f-a92c-4940-a1ce-2c0b3bab1ba5"}
{"level":"error","ts":"2026-07-10T10:53:53.991Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"48fd4e4f-a92c-4940-a1ce-2c0b3bab1ba5","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:54.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"d810ad8b-87df-43fe-a784-020a8d9ec80e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:54.720Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"d810ad8b-87df-43fe-a784-020a8d9ec80e","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:54.720Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"d810ad8b-87df-43fe-a784-020a8d9ec80e","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:54.726Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"d810ad8b-87df-43fe-a784-020a8d9ec80e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:53:55.765Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","enabled":true}
{"level":"error","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7"}
{"level":"error","ts":"2026-07-10T10:53:55.771Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"27e97810-5b1c-4888-a4e8-deb747fd2cc7","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:56.552Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"087cd94f-5d3a-4165-937f-60880eb6b632","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:56.552Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"087cd94f-5d3a-4165-937f-60880eb6b632","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:56.552Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"087cd94f-5d3a-4165-937f-60880eb6b632","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:53:56.554Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"087cd94f-5d3a-4165-937f-60880eb6b632","JobId":"fail-rb92h","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:56.555Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"087cd94f-5d3a-4165-937f-60880eb6b632"}
{"level":"error","ts":"2026-07-10T10:53:56.555Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"087cd94f-5d3a-4165-937f-60880eb6b632","error":"Job fail-rb92h does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:53:57.727Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ed21269c-39c6-42bd-a899-7850896adb70","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:57.727Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ed21269c-39c6-42bd-a899-7850896adb70","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:57.727Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ed21269c-39c6-42bd-a899-7850896adb70","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:57.732Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"ed21269c-39c6-42bd-a899-7850896adb70","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:58.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"7d70a570-ef0b-4033-b3df-cea369652364","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:58.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"7d70a570-ef0b-4033-b3df-cea369652364","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:58.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"7d70a570-ef0b-4033-b3df-cea369652364","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:58.411Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"7d70a570-ef0b-4033-b3df-cea369652364","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:59.416Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"74753cdc-a290-4417-aaeb-c68a6588e4de","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:59.416Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"74753cdc-a290-4417-aaeb-c68a6588e4de","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:59.416Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"74753cdc-a290-4417-aaeb-c68a6588e4de","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:59.419Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"74753cdc-a290-4417-aaeb-c68a6588e4de","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:59.432Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"22cad25a-99fc-4b8b-af42-78780a05fde7","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:59.432Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"22cad25a-99fc-4b8b-af42-78780a05fde7","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:53:59.432Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"22cad25a-99fc-4b8b-af42-78780a05fde7","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:53:59.435Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"22cad25a-99fc-4b8b-af42-78780a05fde7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:53:47Z","endTime":"2026-07-10T10:53:51Z"},"jobId":"counter-5gxzv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-10T10:53:39Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:53:59.435Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"22cad25a-99fc-4b8b-af42-78780a05fde7","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-10T10:53:59.447Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"408c4822-1fbe-4e82-a6da-8d6489370499","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:53:59.447Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"408c4822-1fbe-4e82-a6da-8d6489370499","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:53:59 +0000 UTC","Now":"2026-07-10T10:53:59.447Z","ShutdownTime":"2026-07-10T10:53:59.000Z"}
{"level":"info","ts":"2026-07-10T10:53:59.936Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1ef47d06-3d75-4cb9-833f-fc15d6aaf23b","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:53:59.936Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1ef47d06-3d75-4cb9-833f-fc15d6aaf23b","pod":"long-running-nq5tx-small-group-worker-fwlzv","ip":"10.133.0.50"}
{"level":"info","ts":"2026-07-10T10:53:59.936Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1ef47d06-3d75-4cb9-833f-fc15d6aaf23b"}
{"level":"info","ts":"2026-07-10T10:54:00.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"aea21cd0-5805-442c-831b-51d0376659ae","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:00.733Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"aea21cd0-5805-442c-831b-51d0376659ae","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-10 10:53:59 +0000 UTC","Now":"2026-07-10T10:54:00.733Z","ShutdownTime":"2026-07-10T10:53:59.000Z"}
{"level":"info","ts":"2026-07-10T10:54:01.675Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a9e1d856-4dac-4eeb-94ed-b9a88c114167","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:01.675Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a9e1d856-4dac-4eeb-94ed-b9a88c114167","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:01.675Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a9e1d856-4dac-4eeb-94ed-b9a88c114167","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:01.678Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a9e1d856-4dac-4eeb-94ed-b9a88c114167","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:54:01.678Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"a9e1d856-4dac-4eeb-94ed-b9a88c114167","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:54:01.689Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"830eb5e5-57a8-4372-bd5f-ec2fbb35625c","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:01.689Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"830eb5e5-57a8-4372-bd5f-ec2fbb35625c","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:01.689Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"830eb5e5-57a8-4372-bd5f-ec2fbb35625c","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:01.692Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"830eb5e5-57a8-4372-bd5f-ec2fbb35625c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:54:04.690Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c33f532b-03f2-4fd4-a268-b8c80806d034","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:04.690Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c33f532b-03f2-4fd4-a268-b8c80806d034","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:04.690Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c33f532b-03f2-4fd4-a268-b8c80806d034","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:04.693Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c33f532b-03f2-4fd4-a268-b8c80806d034","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z","endTime":"2026-07-10T10:54:02Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:54:00,682\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-rb92h: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:54:04.693Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"c33f532b-03f2-4fd4-a268-b8c80806d034","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-10T10:54:04.706Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b01eaa36-397f-43d7-adda-f538eac66a80","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:04.706Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b01eaa36-397f-43d7-adda-f538eac66a80","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:04.706Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b01eaa36-397f-43d7-adda-f538eac66a80","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:04.709Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b01eaa36-397f-43d7-adda-f538eac66a80","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z","endTime":"2026-07-10T10:54:02Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:54:00,682\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-rb92h: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z","endTime":"2026-07-10T10:54:02Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:54:00,682\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-rb92h: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-07-10T10:54:05.467Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"e625ff88-ebeb-4b48-8112-2fa3e8a875e5","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:05.467Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"e625ff88-ebeb-4b48-8112-2fa3e8a875e5"}
{"level":"error","ts":"2026-07-10T10:54:05.467Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"e625ff88-ebeb-4b48-8112-2fa3e8a875e5","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.012Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01"}
{"level":"info","ts":"2026-07-10T10:54:06.012Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01"}
{"level":"info","ts":"2026-07-10T10:54:06.012Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:06.012Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01"}
{"level":"info","ts":"2026-07-10T10:54:06.012Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:06.012Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01"}
{"level":"info","ts":"2026-07-10T10:54:06.012Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:06.013Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:06.013Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:54:06.013Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:06.013Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:54:06.013Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:06.013Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:06.013Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01"}
{"level":"error","ts":"2026-07-10T10:54:06.017Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"df516068-7a0c-430a-94ff-c2ff69904a01","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.652Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.652Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T10:54:06.661Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T10:54:06.661Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"09b04865-e24e-4cf3-ad7a-a4429d6ab5b2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.661Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.662Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.662Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.662Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.662Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.667Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.667Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T10:54:06.675Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.678Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.678Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.682Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:06.682Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"09b04865-e24e-4cf3-ad7a-a4429d6ab5b2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.683Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e0fb0ad6-4ebb-4633-9aba-36c707f1e05a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.725Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T10:54:06.725Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.729Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e0fb0ad6-4ebb-4633-9aba-36c707f1e05a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:54:06.735Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.740Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e0fb0ad6-4ebb-4633-9aba-36c707f1e05a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:54:06.740Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e0fb0ad6-4ebb-4633-9aba-36c707f1e05a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.745Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:54:06.825Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.825Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","rayCluster":"ray-cluster","podIPs":[]}
{"level":"error","ts":"2026-07-10T10:54:06.825Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.825Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f"}
{"level":"error","ts":"2026-07-10T10:54:06.825Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"629f8c5e-7654-4d51-9a0d-b3456025189f","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.825Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6a8d5b11-57b9-4875-bed8-d31aca7a0b92","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T10:54:06.834Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:285\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.834Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85f99acb-df42-4baa-9111-4e68208d7230","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.834Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4a237b96-cbb7-449e-b040-3451b05a90a2","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T10:54:06.834Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4a237b96-cbb7-449e-b040-3451b05a90a2","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.834Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4a237b96-cbb7-449e-b040-3451b05a90a2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.834Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4a237b96-cbb7-449e-b040-3451b05a90a2","cluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T10:54:06.835Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6a8d5b11-57b9-4875-bed8-d31aca7a0b92","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:06.835Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6a8d5b11-57b9-4875-bed8-d31aca7a0b92","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.835Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6a8d5b11-57b9-4875-bed8-d31aca7a0b92"}
{"level":"error","ts":"2026-07-10T10:54:06.835Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6a8d5b11-57b9-4875-bed8-d31aca7a0b92","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:06.835Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1bb72e16-5aec-4141-b828-61f5b9e7b6fc","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.835Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1bb72e16-5aec-4141-b828-61f5b9e7b6fc"}
{"level":"info","ts":"2026-07-10T10:54:06.844Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4a237b96-cbb7-449e-b040-3451b05a90a2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.844Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8ab13ebb-9e77-41bc-92bb-22663b51df6b","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.844Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"62409759-f4cb-4cac-b86a-8e0f56279aec","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.844Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8ab13ebb-9e77-41bc-92bb-22663b51df6b"}
{"level":"info","ts":"2026-07-10T10:54:06.846Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2165addf-7ed8-4ec9-bd00-c8a7ebd06219","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.846Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2165addf-7ed8-4ec9-bd00-c8a7ebd06219"}
{"level":"info","ts":"2026-07-10T10:54:06.857Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"62409759-f4cb-4cac-b86a-8e0f56279aec","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:54:06.867Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"62409759-f4cb-4cac-b86a-8e0f56279aec","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:54:06.867Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"62409759-f4cb-4cac-b86a-8e0f56279aec","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:06.920Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.023Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.023Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14"}
{"level":"info","ts":"2026-07-10T10:54:07.023Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14"}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:54:07.029Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14"}
{"level":"error","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c57139a4-b333-4466-b2c6-8b20372aef14","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e"}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e"}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:54:07.037Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:06Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:06Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e"}
{"level":"error","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"53930242-557c-4101-8c49-17cbf7d80b3e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311"}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311"}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:06Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:06Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:07.047Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"931e29cc-da66-4de4-860a-d525e81b8a58","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.048Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2f5e95df-3706-40cc-970e-9da92c39f0af","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.048Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2f5e95df-3706-40cc-970e-9da92c39f0af"}
{"level":"info","ts":"2026-07-10T10:54:07.125Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","roleBinding":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","name":"kuberay-gateway-access","namespace":"test-ns-d4rh6","operation":"created"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311"}
{"level":"error","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"460698ea-74c5-4f0b-8ce5-cdb703eb7311","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491"}
{"level":"error","ts":"2026-07-10T10:54:07.225Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"eec350bf-2f7a-4471-9ea3-5e865c552491","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.234Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","name":"test-ns-d4rh6-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-07-10T10:54:07.235Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"931e29cc-da66-4de4-860a-d525e81b8a58","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:54:07.241Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-07-10T10:54:07.245Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a"}
{"level":"info","ts":"2026-07-10T10:54:07.245Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a"}
{"level":"info","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a"}
{"level":"error","ts":"2026-07-10T10:54:07.246Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6ddfe84c-42b5-43fe-882b-86b550bee91a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.248Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"931e29cc-da66-4de4-860a-d525e81b8a58","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:54:07.248Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"931e29cc-da66-4de4-860a-d525e81b8a58","cluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T10:54:07.325Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.325Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24"}
{"level":"error","ts":"2026-07-10T10:54:07.325Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"81a7003c-0748-4247-8b43-c61e0f831f24","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c"}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c"}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c"}
{"level":"error","ts":"2026-07-10T10:54:07.326Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c9679cd3-1152-4a28-b057-6c0141eef03c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.425Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"097d1d60-1a21-4bf2-84da-f4971088f522","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T10:54:07.425Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"097d1d60-1a21-4bf2-84da-f4971088f522","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.425Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"097d1d60-1a21-4bf2-84da-f4971088f522","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.439Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"097d1d60-1a21-4bf2-84da-f4971088f522","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.439Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6bfe883f-b6da-4108-9f5a-4b0321339d98","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.439Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"eb071a9d-46e3-45a9-8f9c-5211aabd4ba3","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.439Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"eb071a9d-46e3-45a9-8f9c-5211aabd4ba3"}
{"level":"info","ts":"2026-07-10T10:54:07.453Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6bfe883f-b6da-4108-9f5a-4b0321339d98","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:54:07.465Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6bfe883f-b6da-4108-9f5a-4b0321339d98","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:54:07.465Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6bfe883f-b6da-4108-9f5a-4b0321339d98","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09"}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09"}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09"}
{"level":"error","ts":"2026-07-10T10:54:07.488Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8b29768e-dc11-4ac4-b257-ef4454000e09","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.707Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f3218b33-8db2-4bfc-a8c6-267d9fd7b499","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:07.707Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f3218b33-8db2-4bfc-a8c6-267d9fd7b499","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:07.707Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f3218b33-8db2-4bfc-a8c6-267d9fd7b499","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:07.710Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"f3218b33-8db2-4bfc-a8c6-267d9fd7b499","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z","endTime":"2026-07-10T10:54:02Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:54:00,682\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-rb92h: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-10T10:54:00Z","endTime":"2026-07-10T10:54:02Z"},"jobId":"fail-rb92h","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-10 10:54:00,682\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-rb92h: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-10T10:53:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717"}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717"}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717"}
{"level":"error","ts":"2026-07-10T10:54:07.809Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"aff4871b-c5ed-47e0-b820-e994315c0717","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4"}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4"}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4"}
{"level":"error","ts":"2026-07-10T10:54:08.450Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d92be67-e42d-456a-9948-ff9693194ca4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T10:54:09.731Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","enabled":true}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","name":"ray-cluster-head-b4ww4"}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"ray-cluster-head-svc"},"newHead":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-10T10:54:09.748Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"59ced649-86ac-4a1f-ab51-a9368ccec4c6","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"ray-cluster-head-svc"},"newHead":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3fc46bc7-a81d-4d85-97e3-b354a447ea1a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8997c104-d340-4168-85f8-93ec39c3af07","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:09.759Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3fc46bc7-a81d-4d85-97e3-b354a447ea1a","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:54:09.765Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:54:09.765Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51"}
{"level":"error","ts":"2026-07-10T10:54:09.765Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1db772f3-f6a5-4eaa-ad76-86487a21fb51","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:09.770Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8997c104-d340-4168-85f8-93ec39c3af07","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:54:09.770Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8"}
{"level":"info","ts":"2026-07-10T10:54:09.770Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8"}
{"level":"info","ts":"2026-07-10T10:54:09.770Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b6a05c10-0031-4c01-9cdd-41a2b4bf1ec8","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:09.771Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0de56a21-e87c-4a55-8512-42894d776bb8","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:09.783Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8997c104-d340-4168-85f8-93ec39c3af07","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:54:09.783Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8997c104-d340-4168-85f8-93ec39c3af07","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-10T10:54:09.825Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ca0eccd5-175b-414f-830e-a81e7a8143be","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:09.836Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"bcae35d7-c02d-490f-b2a5-36c6ecc30229","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:09.836Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"fa53daf5-68f1-4e37-9851-9c31eb86fe98","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:09.836Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"bcae35d7-c02d-490f-b2a5-36c6ecc30229","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-10T10:54:09.847Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"fa53daf5-68f1-4e37-9851-9c31eb86fe98","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:54:09.858Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"fa53daf5-68f1-4e37-9851-9c31eb86fe98","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:54:09.858Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"fa53daf5-68f1-4e37-9851-9c31eb86fe98","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97"}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97"}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:09.887Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"70731c22-5b3b-4786-b749-76463bc7cf97","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1"}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1"}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:09.892Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"501b83be-83ac-40cf-b8cf-66be96d746e1","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10"}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10"}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"59397861-ef57-4302-82f2-3df926e20e10","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883"}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883"}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-8gjkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:09.925Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b8c27e6-3bc9-4d7c-8f8e-9e7afd9be883","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:10.210Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"58674472-cdeb-48f5-a85b-66c012647715","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:10.367Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1"}
{"level":"info","ts":"2026-07-10T10:54:10.367Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1"}
{"level":"info","ts":"2026-07-10T10:54:10.367Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.367Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.367Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.367Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.367Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","worker Pod":"raycluster-small-group-worker-8gjkr","shouldDelete":true,"reason":"The worker Pod raycluster-small-group-worker-8gjkr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T10:54:10.371Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.371Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-07-10T10:54:10.371Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.371Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:37:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:10.383Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1"}
{"level":"error","ts":"2026-07-10T10:54:10.383Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b33cea69-322b-413e-8c02-43235135dae1","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.383Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"fdde8551-bcd4-49dd-b024-a542aabff80a","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.383Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"68d450de-e480-450a-8286-d7e5ec1a1981","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.384Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"68d450de-e480-450a-8286-d7e5ec1a1981","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:54:10.384Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"68d450de-e480-450a-8286-d7e5ec1a1981","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:54:10.384Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"68d450de-e480-450a-8286-d7e5ec1a1981","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:54:10.388Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3"}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3"}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3","podName":"raycluster-small-group-worker-8gjkr","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"fdde8551-bcd4-49dd-b024-a542aabff80a","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:10.389Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"1db6088b-8eed-4665-8a8b-5f67296c14f3","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:10.433Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4006d7dd-803a-4e25-9853-544d402af9d3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4006d7dd-803a-4e25-9853-544d402af9d3","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:54:10.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4006d7dd-803a-4e25-9853-544d402af9d3","pod":"raycluster-small-group-worker-8gjkr","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-10T10:54:10.433Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4006d7dd-803a-4e25-9853-544d402af9d3","rayCluster":"raycluster","podIPs":["10.134.0.17","10.133.0.32"]}
{"level":"info","ts":"2026-07-10T10:54:10.468Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c"}
{"level":"info","ts":"2026-07-10T10:54:10.468Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c"}
{"level":"info","ts":"2026-07-10T10:54:10.468Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:54:10.469Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c","podName":"raycluster-small-group-worker-8gjkr","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:54:10.469Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:10.469Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3b433fd2-7968-45d0-b562-5471ba32a04c","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a"}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a"}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a"}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.472Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.475Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.475Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.476Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:54:10.476Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:10.487Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a"}
{"level":"error","ts":"2026-07-10T10:54:10.487Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"20bd115b-1d9a-4e5a-ba9b-6b69d2a8af2a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.487Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c59b831e-92cb-4626-a206-c4fbc5261880","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.487Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e7fbf71f-049b-4283-a3fa-922eec3f54d1","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.487Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c59b831e-92cb-4626-a206-c4fbc5261880","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e7fbf71f-049b-4283-a3fa-922eec3f54d1","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a"}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.492Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.495Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.495Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.496Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a"}
{"level":"error","ts":"2026-07-10T10:54:10.496Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"b4997ae0-3ef0-41e2-ab15-7cf08b7fcb5a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.497Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c59b831e-92cb-4626-a206-c4fbc5261880","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-xxj65.svc","raycluster-head-svc.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["10.134.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f"}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f"}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c59b831e-92cb-4626-a206-c4fbc5261880","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-xxj65.svc","raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*.raycluster-worker-svc.test-ns-xxj65.svc","*.raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*-worker-*.test-ns-xxj65.svc","*-worker-*.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["10.134.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c59b831e-92cb-4626-a206-c4fbc5261880","rayCluster":"raycluster","podIPs":["10.134.0.17"]}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f"}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.525Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.529Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.529Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.530Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f"}
{"level":"error","ts":"2026-07-10T10:54:10.530Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"77dcde8e-1733-4f95-b96a-41af415bc33f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","head Pod":"raycluster-head-j5zbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-j5zbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292"}
{"level":"info","ts":"2026-07-10T10:54:10.550Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.551Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.551Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.551Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.554Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.554Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.554Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292"}
{"level":"error","ts":"2026-07-10T10:54:10.554Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"c1ecdee1-7452-4291-8acb-46b9daefd292","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.561Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ccde40fe-c916-46ed-bc0c-ae4ecd464610"}
{"level":"info","ts":"2026-07-10T10:54:10.561Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ccde40fe-c916-46ed-bc0c-ae4ecd464610"}
{"level":"info","ts":"2026-07-10T10:54:10.561Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ccde40fe-c916-46ed-bc0c-ae4ecd464610","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.561Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ccde40fe-c916-46ed-bc0c-ae4ecd464610","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:10.561Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ccde40fe-c916-46ed-bc0c-ae4ecd464610","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:10.561Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ccde40fe-c916-46ed-bc0c-ae4ecd464610","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"newHead":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-10T10:54:10.561Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ccde40fe-c916-46ed-bc0c-ae4ecd464610","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:10.625Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"df0e96df-2c8d-437b-9759-f95e9c61aec8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:10.625Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"13b8fd6a-ae71-4b70-bb0e-e7e5768cb6a1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:10.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7"}
{"level":"info","ts":"2026-07-10T10:54:10.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7"}
{"level":"info","ts":"2026-07-10T10:54:10.625Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"df0e96df-2c8d-437b-9759-f95e9c61aec8","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:54:10.626Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.626Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7"}
{"level":"info","ts":"2026-07-10T10:54:10.626Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.626Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.626Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.626Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.629Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.629Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.630Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7"}
{"level":"error","ts":"2026-07-10T10:54:10.630Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"bf726913-e407-4f30-bb8b-a7e6823b50c7","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.637Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"13b8fd6a-ae71-4b70-bb0e-e7e5768cb6a1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:54:10.637Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"df0e96df-2c8d-437b-9759-f95e9c61aec8","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-d4rh6.svc","ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:54:10.646Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"df0e96df-2c8d-437b-9759-f95e9c61aec8","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-d4rh6.svc","ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-d4rh6.svc","*.ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*-worker-*.test-ns-d4rh6.svc","*-worker-*.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:54:10.647Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"df0e96df-2c8d-437b-9759-f95e9c61aec8","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:54:10.648Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"13b8fd6a-ae71-4b70-bb0e-e7e5768cb6a1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:54:10.648Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"13b8fd6a-ae71-4b70-bb0e-e7e5768cb6a1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3185a2fc-e465-4d8c-9f69-b1b872effb97","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3185a2fc-e465-4d8c-9f69-b1b872effb97","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3185a2fc-e465-4d8c-9f69-b1b872effb97","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e"}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e"}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e"}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.711Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.725Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.725Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.725Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e"}
{"level":"error","ts":"2026-07-10T10:54:10.725Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8fdf3471-2334-4d82-81d0-31070604442e","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.752Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90"}
{"level":"info","ts":"2026-07-10T10:54:10.752Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90"}
{"level":"info","ts":"2026-07-10T10:54:10.752Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","Found 1 head Pod":"raycluster-head-j5zbf","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-10 10:37:34 +0000 UTC,FinishedAt:2026-07-10 10:54:10 +0000 UTC,ContainerID:cri-o://cacfb8d0e1e78815c30ea60fad2fbfc8528dfa0d5656c28a1acc70e1122d19ff,}"}
{"level":"info","ts":"2026-07-10T10:54:10.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","head Pod":"raycluster-head-j5zbf","shouldDelete":true,"reason":"The head Pod raycluster-head-j5zbf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T10:54:10.756Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-head-j5zbf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.756Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","error":"The head Pod raycluster-head-j5zbf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-10T10:54:10.756Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}]}
{"level":"info","ts":"2026-07-10T10:54:10.756Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:37:49Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:10.768Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90"}
{"level":"error","ts":"2026-07-10T10:54:10.768Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"ab7d2bec-419e-4d0f-8b7b-5600d9e9ce90","error":"The head Pod raycluster-head-j5zbf status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.768Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"71b91a66-7d77-4bc0-a8c6-82eb4f7fb448","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.768Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f4ab86ee-ad80-4e74-8ce3-3f62c6e3f014","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.768Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"71b91a66-7d77-4bc0-a8c6-82eb4f7fb448","pod":"raycluster-head-j5zbf","ip":"10.134.0.17"}
{"level":"info","ts":"2026-07-10T10:54:10.768Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"71b91a66-7d77-4bc0-a8c6-82eb4f7fb448"}
{"level":"info","ts":"2026-07-10T10:54:10.773Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f4ab86ee-ad80-4e74-8ce3-3f62c6e3f014","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414"}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.886Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.890Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.890Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.890Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414"}
{"level":"error","ts":"2026-07-10T10:54:10.890Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"8ee816f6-3bf0-4a08-8dfb-2f4b342a7414","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.899Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a"}
{"level":"info","ts":"2026-07-10T10:54:10.899Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a"}
{"level":"info","ts":"2026-07-10T10:54:10.899Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a"}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:54:10.900Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.903Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.903Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.903Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a"}
{"level":"error","ts":"2026-07-10T10:54:10.903Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"f3feb1e6-7df0-4233-ab8f-3eee6836419a","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:10.904Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:10.907Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.907Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:10.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:10.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:10.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:10.907Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"newHead":{"serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-10T10:54:10.907Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"raycluster-head-j5zbf","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","stateTransitionTimes":{"ready":"2026-07-10T10:37:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:37:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:33:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:33:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T10:54:10Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:54:10.925Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5"}
{"level":"error","ts":"2026-07-10T10:54:10.925Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"7f674e14-1fd2-48c5-967a-4b380d80b6d5","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:10.925Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c1d73b93-7d1b-4c30-99cf-410ed2624699","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.925Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"648037dc-95e8-4991-a9bc-ee8005d023d1","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.931Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c1d73b93-7d1b-4c30-99cf-410ed2624699","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:10.936Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"648037dc-95e8-4991-a9bc-ee8005d023d1","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-xxj65.svc","raycluster-head-svc.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:54:10.948Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"648037dc-95e8-4991-a9bc-ee8005d023d1","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-xxj65.svc","raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*.raycluster-worker-svc.test-ns-xxj65.svc","*.raycluster-worker-svc.test-ns-xxj65.svc.cluster.local","*-worker-*.test-ns-xxj65.svc","*-worker-*.test-ns-xxj65.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:54:10.948Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"648037dc-95e8-4991-a9bc-ee8005d023d1"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-xxj65.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-xxj65.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:11.531Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:11.535Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:11.535Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:11.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:11.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:11.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:11.535Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1"}
{"level":"error","ts":"2026-07-10T10:54:11.535Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"2ec2064e-3bcc-4c78-8435-c5f1ea79b6f1","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:11.824Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc"}
{"level":"error","ts":"2026-07-10T10:54:11.827Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"ce320558-7988-4363-b561-7947ead9ccfc","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:54:11.837Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05a202c5-dc13-4f3a-bc1c-808aab64dd78","seconds":300}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:12.521Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:12.527Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:12.527Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:12.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.528Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:54:12.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.528Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd"}
{"level":"error","ts":"2026-07-10T10:54:12.528Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"75e3e54a-3a5c-46f4-9117-7e8d9a026cfd","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:12.538Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:12.541Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:12.541Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:12.541Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.541Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","podName":"long-running-nq5tx-small-group-worker-fwlzv","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T10:54:12.541Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.541Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.542Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6"}
{"level":"error","ts":"2026-07-10T10:54:12.542Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"e7c76222-e81e-4a91-b6e1-40fa7270cec6","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","1 head service found":"long-running-nq5tx-head-svc"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","mode":"IntegratedOAuth","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","cluster":"long-running-nq5tx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","cluster":"long-running-nq5tx","authType":"IntegratedOAuth","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-nq5tx","ray.io/group":"headgroup","ray.io/identifier":"long-running-nq5tx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-nq5tx-head-svc.test-ns-z4qpg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-10T10:54:12.545Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","enabled":true}
{"level":"error","ts":"2026-07-10T10:54:12.548Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:12.548Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:12.548Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.548Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.548Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","filter labels":["test-ns-z4qpg",{"ray.io/cluster":"long-running-nq5tx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:12.548Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0"}
{"level":"error","ts":"2026-07-10T10:54:12.548Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"474743b5-4d86-42ca-a5d0-4d98d04ac3b0","error":"FailedCreateHeadPod\npods \"long-running-nq5tx-head-\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:12.712Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3185a2fc-e465-4d8c-9f69-b1b872effb97","JobId":"fail-rb92h","error":"Get \"http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265/api/jobs/fail-rb92h\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:12.712Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3185a2fc-e465-4d8c-9f69-b1b872effb97"}
{"level":"error","ts":"2026-07-10T10:54:12.712Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3185a2fc-e465-4d8c-9f69-b1b872effb97","error":"Get \"http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265/api/jobs/fail-rb92h\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:12.712Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b6675ede-460a-4162-91a1-1d8b018c4e4d","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:12.712Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b6675ede-460a-4162-91a1-1d8b018c4e4d","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:12.712Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b6675ede-460a-4162-91a1-1d8b018c4e4d","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"error","ts":"2026-07-10T10:54:14.713Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b6675ede-460a-4162-91a1-1d8b018c4e4d","JobId":"fail-rb92h","error":"Get \"http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265/api/jobs/fail-rb92h\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:14.713Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b6675ede-460a-4162-91a1-1d8b018c4e4d"}
{"level":"error","ts":"2026-07-10T10:54:14.713Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b6675ede-460a-4162-91a1-1d8b018c4e4d","error":"Get \"http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265/api/jobs/fail-rb92h\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:14.713Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3be0e7e5-c68a-4499-841a-b48d63f486ce","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-10T10:54:14.713Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3be0e7e5-c68a-4499-841a-b48d63f486ce","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:14.713Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3be0e7e5-c68a-4499-841a-b48d63f486ce","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:14.930Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"cd32f726-d1b2-46fd-8421-cbe50dd1c8fe","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:14.930Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"cd32f726-d1b2-46fd-8421-cbe50dd1c8fe","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:14.930Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"cd32f726-d1b2-46fd-8421-cbe50dd1c8fe","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:14.930Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb"}
{"level":"info","ts":"2026-07-10T10:54:14.930Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb"}
{"level":"error","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"raycluster-head-svc\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb","error":"services \"raycluster-head-svc\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated"}
{"level":"info","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb","serviceSelectors":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/identifier":"raycluster-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb","filter labels":["test-ns-xxj65",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb","error":"unable to find head service. cluster name raycluster, filter labels [test-ns-xxj65 map[ray.io/cluster:raycluster ray.io/identifier:raycluster-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb"}
{"level":"error","ts":"2026-07-10T10:54:14.933Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"a4c0a5e5-70ca-41da-8150-4045ba55dcbb","error":"services \"raycluster-head-svc\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:14.939Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"cd32f726-d1b2-46fd-8421-cbe50dd1c8fe","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.077Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b51375e3-25dc-4c0d-924f-de3a5d921290","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.077Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b51375e3-25dc-4c0d-924f-de3a5d921290","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.077Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b51375e3-25dc-4c0d-924f-de3a5d921290","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.090Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b51375e3-25dc-4c0d-924f-de3a5d921290","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.091Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b51375e3-25dc-4c0d-924f-de3a5d921290"}
{"level":"error","ts":"2026-07-10T10:54:15.091Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"b51375e3-25dc-4c0d-924f-de3a5d921290","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.091Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8427bfd2-5d3d-43bc-a9e8-a496e6943a12","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.091Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"858fce42-3451-4b6e-96ec-317ad0349d38","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.091Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"8427bfd2-5d3d-43bc-a9e8-a496e6943a12"}
{"level":"info","ts":"2026-07-10T10:54:15.096Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8ee3bd6-dd85-4347-b64a-aafebfa07a7d","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.096Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8ee3bd6-dd85-4347-b64a-aafebfa07a7d","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.096Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8ee3bd6-dd85-4347-b64a-aafebfa07a7d","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.096Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"858fce42-3451-4b6e-96ec-317ad0349d38","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.106Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8ee3bd6-dd85-4347-b64a-aafebfa07a7d","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.106Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8ee3bd6-dd85-4347-b64a-aafebfa07a7d"}
{"level":"error","ts":"2026-07-10T10:54:15.106Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"a8ee3bd6-dd85-4347-b64a-aafebfa07a7d","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.116Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"3aededc3-ebe3-4466-8f08-015e917f4e02","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.116Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"3aededc3-ebe3-4466-8f08-015e917f4e02","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.116Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"3aededc3-ebe3-4466-8f08-015e917f4e02","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.128Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"3aededc3-ebe3-4466-8f08-015e917f4e02","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.129Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"3aededc3-ebe3-4466-8f08-015e917f4e02"}
{"level":"error","ts":"2026-07-10T10:54:15.129Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"3aededc3-ebe3-4466-8f08-015e917f4e02","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.225Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c44475e7-9564-4da5-b044-1800be4193c1","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.225Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c44475e7-9564-4da5-b044-1800be4193c1","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.225Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c44475e7-9564-4da5-b044-1800be4193c1","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.236Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c44475e7-9564-4da5-b044-1800be4193c1","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.236Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c44475e7-9564-4da5-b044-1800be4193c1"}
{"level":"error","ts":"2026-07-10T10:54:15.236Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"c44475e7-9564-4da5-b044-1800be4193c1","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.276Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ef693cb3-a587-48b6-b5be-6c8b14c048a1","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.276Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ef693cb3-a587-48b6-b5be-6c8b14c048a1","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.276Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ef693cb3-a587-48b6-b5be-6c8b14c048a1","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.287Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ef693cb3-a587-48b6-b5be-6c8b14c048a1","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.287Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ef693cb3-a587-48b6-b5be-6c8b14c048a1"}
{"level":"error","ts":"2026-07-10T10:54:15.287Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ef693cb3-a587-48b6-b5be-6c8b14c048a1","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.367Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ca68398e-9e45-47f7-a691-b6cb7982ad4f","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.367Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ca68398e-9e45-47f7-a691-b6cb7982ad4f","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.367Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ca68398e-9e45-47f7-a691-b6cb7982ad4f","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.379Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ca68398e-9e45-47f7-a691-b6cb7982ad4f","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.379Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ca68398e-9e45-47f7-a691-b6cb7982ad4f"}
{"level":"error","ts":"2026-07-10T10:54:15.379Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"ca68398e-9e45-47f7-a691-b6cb7982ad4f","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.539Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33d96504-b178-4dc6-b22e-eea215807a92","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.539Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33d96504-b178-4dc6-b22e-eea215807a92","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.539Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33d96504-b178-4dc6-b22e-eea215807a92","cluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.550Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33d96504-b178-4dc6-b22e-eea215807a92","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.550Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33d96504-b178-4dc6-b22e-eea215807a92"}
{"level":"error","ts":"2026-07-10T10:54:15.550Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"33d96504-b178-4dc6-b22e-eea215807a92","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.650Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0f9cfd11-3f6b-476f-b699-17458cbf2f9f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.656Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0f9cfd11-3f6b-476f-b699-17458cbf2f9f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.656Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f100e112-308d-4342-b031-bfad1e0a8d80","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.661Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f100e112-308d-4342-b031-bfad1e0a8d80","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"22e5c53b-7f48-4752-92da-ba8b7bef5161","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"91168ca2-d2fb-44ee-b5c7-db31702570b5","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"84027e97-5a71-4f95-9236-95c539a08efc"}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"91168ca2-d2fb-44ee-b5c7-db31702570b5","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"reconcileID":"3a50ba33-b2f8-4ff3-a27a-0be7fef2412d"}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"22e5c53b-7f48-4752-92da-ba8b7bef5161","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"22e5c53b-7f48-4752-92da-ba8b7bef5161","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.679Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"22e5c53b-7f48-4752-92da-ba8b7bef5161","namespace":"test-ns-xxj65","otherAuthClusters":0,"action":"will delete"}
{"level":"error","ts":"2026-07-10T10:54:15.682Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"91168ca2-d2fb-44ee-b5c7-db31702570b5","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:15.682Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"91168ca2-d2fb-44ee-b5c7-db31702570b5","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.682Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"91168ca2-d2fb-44ee-b5c7-db31702570b5"}
{"level":"error","ts":"2026-07-10T10:54:15.682Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"91168ca2-d2fb-44ee-b5c7-db31702570b5","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.687Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"22e5c53b-7f48-4752-92da-ba8b7bef5161","name":"kuberay-gateway-access","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:54:15.688Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d976231a-e813-4512-aa24-730364f63925","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.688Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d976231a-e813-4512-aa24-730364f63925","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.690Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d976231a-e813-4512-aa24-730364f63925","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:15.690Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d976231a-e813-4512-aa24-730364f63925","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.690Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d976231a-e813-4512-aa24-730364f63925"}
{"level":"error","ts":"2026-07-10T10:54:15.690Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"d976231a-e813-4512-aa24-730364f63925","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.692Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"22e5c53b-7f48-4752-92da-ba8b7bef5161","name":"test-ns-xxj65-raycluster","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T10:54:15.699Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.699Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-xxj65"}
{"level":"info","ts":"2026-07-10T10:54:15.701Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9f92ef63-628b-48ea-9cbe-465491e78b7c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.701Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9f92ef63-628b-48ea-9cbe-465491e78b7c","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-10T10:54:15.705Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9f92ef63-628b-48ea-9cbe-465491e78b7c","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:15.705Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9f92ef63-628b-48ea-9cbe-465491e78b7c","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.705Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9f92ef63-628b-48ea-9cbe-465491e78b7c"}
{"level":"error","ts":"2026-07-10T10:54:15.705Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"9f92ef63-628b-48ea-9cbe-465491e78b7c","error":"certificates.cert-manager.io \"ray-ca-certificate-raycluster\" is forbidden: unable to create new content in namespace test-ns-xxj65 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.712Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"22e5c53b-7f48-4752-92da-ba8b7bef5161","cluster":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.714Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2a19681c-41ce-4b1b-9ed8-16d7e14f677b","namespace":"test-ns-xxj65","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.714Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"0509682a-087c-421e-ad5b-9d8bc5af2399"}
{"level":"info","ts":"2026-07-10T10:54:15.714Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4aa091cb-a7fd-4417-bb72-ed934135c87f","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.714Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4aa091cb-a7fd-4417-bb72-ed934135c87f"}
{"level":"info","ts":"2026-07-10T10:54:15.714Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4aa091cb-a7fd-4417-bb72-ed934135c87f","namespace":"test-ns-xxj65","authClusters":0}
{"level":"info","ts":"2026-07-10T10:54:15.714Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4aa091cb-a7fd-4417-bb72-ed934135c87f","namespace":"test-ns-xxj65"}
{"level":"error","ts":"2026-07-10T10:54:15.722Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2a19681c-41ce-4b1b-9ed8-16d7e14f677b","issuer":"ray-selfsigned-issuer-raycluster","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.737Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2a19681c-41ce-4b1b-9ed8-16d7e14f677b","errorCount":1}
{"level":"info","ts":"2026-07-10T10:54:15.737Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2a19681c-41ce-4b1b-9ed8-16d7e14f677b"}
{"level":"error","ts":"2026-07-10T10:54:15.737Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"2a19681c-41ce-4b1b-9ed8-16d7e14f677b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:15.737Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f60d2c38-31f1-4641-986b-95b067c7f319","namespace":"test-ns-xxj65","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.737Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"f60d2c38-31f1-4641-986b-95b067c7f319","namespace":"test-ns-xxj65","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.777Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4a14171c-79e9-4cec-9afa-46aebf855269","namespace":"test-ns-xxj65","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.777Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"4a14171c-79e9-4cec-9afa-46aebf855269","namespace":"test-ns-xxj65","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-10T10:54:15.871Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e8337fe2-4a6a-4f91-a4ee-148cbb827bd2","namespacedName":{"name":"raycluster","namespace":"test-ns-xxj65"}}
{"level":"info","ts":"2026-07-10T10:54:15.871Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e8337fe2-4a6a-4f91-a4ee-148cbb827bd2"}
{"level":"info","ts":"2026-07-10T10:54:15.871Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e8337fe2-4a6a-4f91-a4ee-148cbb827bd2","namespace":"test-ns-xxj65","authClusters":0}
{"level":"info","ts":"2026-07-10T10:54:15.871Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-xxj65"},"namespace":"test-ns-xxj65","name":"raycluster","reconcileID":"e8337fe2-4a6a-4f91-a4ee-148cbb827bd2","namespace":"test-ns-xxj65"}
{"level":"error","ts":"2026-07-10T10:54:16.714Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3be0e7e5-c68a-4499-841a-b48d63f486ce","JobId":"fail-rb92h","error":"Get \"http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265/api/jobs/fail-rb92h\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:16.714Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3be0e7e5-c68a-4499-841a-b48d63f486ce"}
{"level":"error","ts":"2026-07-10T10:54:16.714Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"3be0e7e5-c68a-4499-841a-b48d63f486ce","error":"Get \"http://raycluster-head-svc.test-ns-xxj65.svc.cluster.local:8265/api/jobs/fail-rb92h\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:16.714Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4c87dc32-a8fc-4716-9bb5-2f9ad26ed118","DeletionTimestamp":"2026-07-10 10:54:15 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:54:16.714Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4c87dc32-a8fc-4716-9bb5-2f9ad26ed118","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:54:16.728Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"managed-externally","namespace":"test-ns-xxj65"},"reconcileID":"4d2e3b9b-d09c-4669-a6b0-32db64a871a4"}
{"level":"info","ts":"2026-07-10T10:54:16.728Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"20b1439e-dae0-48d2-bdb9-5dc0e922a0f5","DeletionTimestamp":"2026-07-10 10:54:15 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:54:16.728Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"20b1439e-dae0-48d2-bdb9-5dc0e922a0f5","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:54:16.739Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"0a6dae69-6bf7-4758-8f66-b951dbadf77a"}
{"level":"info","ts":"2026-07-10T10:54:16.739Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b2067e35-510a-483e-b08a-63e34ca777ea","DeletionTimestamp":"2026-07-10 10:54:15 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:54:16.739Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b2067e35-510a-483e-b08a-63e34ca777ea","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-07-10T10:54:16.743Z","logger":"controllers.RayJob","msg":"Failed to remove finalizer for RayJob","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b2067e35-510a-483e-b08a-63e34ca777ea","error":"rayjobs.ray.io \"fail\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:130\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:16.743Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b2067e35-510a-483e-b08a-63e34ca777ea"}
{"level":"error","ts":"2026-07-10T10:54:16.743Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"b2067e35-510a-483e-b08a-63e34ca777ea","error":"rayjobs.ray.io \"fail\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:16.743Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"0b6c92ad-4f75-4f97-855c-b5dc00d1064e"}
{"level":"info","ts":"2026-07-10T10:54:16.748Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-xxj65"},"reconcileID":"7af11ada-7d68-4db7-be3e-c4165b84acfb"}
{"level":"info","ts":"2026-07-10T10:54:19.729Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-xxj65"},"reconcileID":"4eeb29ee-8be5-4434-a93a-ece6fc779e33"}
{"level":"info","ts":"2026-07-10T10:54:20.452Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"94612fad-d7b9-453c-814d-ee8717017eab"}
{"level":"info","ts":"2026-07-10T10:54:20.458Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"61f607d1-413e-4e15-a188-00606eb898c9","DeletionTimestamp":"2026-07-10 10:54:20 +0000 UTC"}
{"level":"info","ts":"2026-07-10T10:54:20.458Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"61f607d1-413e-4e15-a188-00606eb898c9","rayJob":"long-running-mk9bd"}
{"level":"error","ts":"2026-07-10T10:54:20.458Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"61f607d1-413e-4e15-a188-00606eb898c9","error":"Post \"http:///api/jobs/long-running-mk9bd/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.458Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"61f607d1-413e-4e15-a188-00606eb898c9","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-10T10:54:20.472Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"def1fab6-9826-495c-8587-bccf456f57e1"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"reconcileID":"cdcfa5d9-03c7-401f-b662-9e9a7c5003a5"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"3c5c6a8a-1e79-4992-899a-a73d64e066e7"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6a04bb4f-6d5e-454e-82ed-11aa0b3ff5d0","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6a04bb4f-6d5e-454e-82ed-11aa0b3ff5d0","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:54:20.482Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"b720d789-8cfa-4beb-a120-1848748ca2f4"}
{"level":"error","ts":"2026-07-10T10:54:20.485Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6a04bb4f-6d5e-454e-82ed-11aa0b3ff5d0","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:20.485Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6a04bb4f-6d5e-454e-82ed-11aa0b3ff5d0","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.485Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6a04bb4f-6d5e-454e-82ed-11aa0b3ff5d0"}
{"level":"error","ts":"2026-07-10T10:54:20.485Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6a04bb4f-6d5e-454e-82ed-11aa0b3ff5d0","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.486Z","logger":"authentication-controller","msg":"Failed to cleanup ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","error":"failed to delete ReferenceGrant: referencegrants.gateway.networking.k8s.io \"kuberay-gateway-access\" not found"}
{"level":"info","ts":"2026-07-10T10:54:20.490Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"7f2606ad-b237-47d5-b154-adba068f845b","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.490Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"7f2606ad-b237-47d5-b154-adba068f845b","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.491Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","name":"test-ns-z4qpg-long-running-nq5tx","namespace":"opendatahub"}
{"level":"error","ts":"2026-07-10T10:54:20.493Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"7f2606ad-b237-47d5-b154-adba068f845b","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:20.493Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"7f2606ad-b237-47d5-b154-adba068f845b","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.493Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"7f2606ad-b237-47d5-b154-adba068f845b"}
{"level":"error","ts":"2026-07-10T10:54:20.493Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"7f2606ad-b237-47d5-b154-adba068f845b","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.497Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","serviceAccount":"long-running-nq5tx-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T10:54:20.503Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"de3306f6-1500-47b3-8c7f-60506cdcc53b","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.503Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"de3306f6-1500-47b3-8c7f-60506cdcc53b","rayCluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.503Z","logger":"raycluster-resource","msg":"default","name":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.503Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}
{"level":"error","ts":"2026-07-10T10:54:20.506Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"de3306f6-1500-47b3-8c7f-60506cdcc53b","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T10:54:20.506Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"de3306f6-1500-47b3-8c7f-60506cdcc53b","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.506Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"de3306f6-1500-47b3-8c7f-60506cdcc53b"}
{"level":"error","ts":"2026-07-10T10:54:20.506Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"de3306f6-1500-47b3-8c7f-60506cdcc53b","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-nq5tx\" is forbidden: unable to create new content in namespace test-ns-z4qpg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"5ee3e5b3-9810-46fa-b126-6aeb1f5a228a","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30","cluster":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30","namespace":"test-ns-z4qpg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"d54b118c-06aa-4634-bb8f-2295116271ab"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"555d4e65-ab0d-42e9-a426-a78fb43c65e2","namespace":"test-ns-z4qpg","clusterName":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.512Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"7259262d-4f16-4765-82a3-1607ff7532c3"}
{"level":"error","ts":"2026-07-10T10:54:20.515Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30","error":"rayclusters.ray.io \"long-running-nq5tx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.515Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30"}
{"level":"error","ts":"2026-07-10T10:54:20.515Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"b3aa2bf7-b000-4106-920d-54a722b60f30","error":"rayclusters.ray.io \"long-running-nq5tx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.515Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ea1e8174-98ad-48bc-9044-a33a3461d456","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:54:20.515Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ea1e8174-98ad-48bc-9044-a33a3461d456"}
{"level":"info","ts":"2026-07-10T10:54:20.515Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ea1e8174-98ad-48bc-9044-a33a3461d456","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:54:20.515Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"ea1e8174-98ad-48bc-9044-a33a3461d456","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:54:20.518Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"978f693e-7b4c-4fda-9912-4ab6434a4fe6"}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1b2fe7b3-29f5-42c2-a3b2-3d8d34a34adf","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1b2fe7b3-29f5-42c2-a3b2-3d8d34a34adf"}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1b2fe7b3-29f5-42c2-a3b2-3d8d34a34adf","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"1b2fe7b3-29f5-42c2-a3b2-3d8d34a34adf","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"fd466850-34d6-4532-8eab-141e9dc12cfa","namespacedName":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"}}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"fd466850-34d6-4532-8eab-141e9dc12cfa"}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"fd466850-34d6-4532-8eab-141e9dc12cfa","namespace":"test-ns-z4qpg","authClusters":0}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"fd466850-34d6-4532-8eab-141e9dc12cfa","namespace":"test-ns-z4qpg"}
{"level":"info","ts":"2026-07-10T10:54:20.520Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"bfae21cd-3e56-460f-a657-fd835badf582"}
{"level":"error","ts":"2026-07-10T10:54:20.522Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"555d4e65-ab0d-42e9-a426-a78fb43c65e2","issuer":"ray-selfsigned-issuer-long-running-nq5tx","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-nq5tx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.536Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"555d4e65-ab0d-42e9-a426-a78fb43c65e2","errorCount":1}
{"level":"info","ts":"2026-07-10T10:54:20.536Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"555d4e65-ab0d-42e9-a426-a78fb43c65e2"}
{"level":"error","ts":"2026-07-10T10:54:20.536Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"555d4e65-ab0d-42e9-a426-a78fb43c65e2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-nq5tx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:54:20.536Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"12cc8889-518f-445d-ab1c-9fdde4787ae9","namespace":"test-ns-z4qpg","clusterName":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.536Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"12cc8889-518f-445d-ab1c-9fdde4787ae9","namespace":"test-ns-z4qpg","clusterName":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.576Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6595445b-4851-4709-b581-27727f59dbc7","namespace":"test-ns-z4qpg","clusterName":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:20.576Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-nq5tx","namespace":"test-ns-z4qpg"},"namespace":"test-ns-z4qpg","name":"long-running-nq5tx","reconcileID":"6595445b-4851-4709-b581-27727f59dbc7","namespace":"test-ns-z4qpg","clusterName":"long-running-nq5tx"}
{"level":"info","ts":"2026-07-10T10:54:23.471Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-z4qpg"},"reconcileID":"65d2bfe3-2c27-48e1-8c6d-9cf08c5c608d"}
{"level":"info","ts":"2026-07-10T10:54:36.846Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"13248b91-22e4-4a15-b486-484e310ee5b1","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:54:36.846Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"13248b91-22e4-4a15-b486-484e310ee5b1","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:54:36.846Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"13248b91-22e4-4a15-b486-484e310ee5b1","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:54:46.427Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-z4qpg"},"reconcileID":"6ddea17d-ba8a-4eda-a04b-18d58af9c820"}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701"}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701"}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:35.894Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:55:35.895Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:55:35.895Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:55:35.895Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cb5b3433-0216-4752-bb12-8f03442a7701","seconds":300}
{"level":"info","ts":"2026-07-10T10:55:36.847Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"103f3147-4123-41f8-bcc7-b2f437b8d98c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:55:36.847Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"103f3147-4123-41f8-bcc7-b2f437b8d98c","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:55:36.847Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"103f3147-4123-41f8-bcc7-b2f437b8d98c","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942"}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942"}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","podName":"ray-cluster-head-b4ww4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:54:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","podName":"ray-cluster-head-b4ww4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:54:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-10T10:55:36.904Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b3fed972-7717-4b38-a9e3-6ea9c0d00942","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:54:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:55:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:55:36.917Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"da2109e1-5f0c-406b-ae77-91e75ca9fa6e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:55:36.917Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ca616e19-fec5-47d3-91bb-3d42bc5efe81","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:55:36.917Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"da2109e1-5f0c-406b-ae77-91e75ca9fa6e","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:55:36.917Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"da2109e1-5f0c-406b-ae77-91e75ca9fa6e","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:55:36.934Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ca616e19-fec5-47d3-91bb-3d42bc5efe81","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:55:36.945Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ca616e19-fec5-47d3-91bb-3d42bc5efe81","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:55:36.945Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ca616e19-fec5-47d3-91bb-3d42bc5efe81","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:55:38.917Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1"}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1"}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:38.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","podName":"ray-cluster-head-b4ww4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:54:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:55:38.919Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","podName":"ray-cluster-head-b4ww4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:54:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T10:55:38.919Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:55:38.919Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"35f31f3d-1749-4050-94bc-1f553fe2f6f1","seconds":300}
{"level":"info","ts":"2026-07-10T10:55:57.936Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543"}
{"level":"info","ts":"2026-07-10T10:55:57.936Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543"}
{"level":"info","ts":"2026-07-10T10:55:57.936Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T10:55:57.937Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cde48fb1-1e8d-4e7f-a89e-5bb92e8b6543","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:55:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:55:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T10:55:57.949Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"21d021fe-b0d0-4f76-962a-9c60bffc90a6","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:55:57.949Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"42b100f7-bf73-4300-a053-4f698ce6edd8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:55:57.949Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"21d021fe-b0d0-4f76-962a-9c60bffc90a6","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:55:57.949Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"21d021fe-b0d0-4f76-962a-9c60bffc90a6","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:55:57.964Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"42b100f7-bf73-4300-a053-4f698ce6edd8","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:55:57.975Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"42b100f7-bf73-4300-a053-4f698ce6edd8","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:55:57.975Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"42b100f7-bf73-4300-a053-4f698ce6edd8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd"}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd"}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:55:59.949Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bd48c18d-9e09-4f17-8866-037e2254b4dd","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:04.447Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.447Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85916a7f-27f2-4894-9526-ce0ba8ebc76a","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3bb8e0c0-eed5-4fb5-a727-d30806d584a2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85916a7f-27f2-4894-9526-ce0ba8ebc76a","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85916a7f-27f2-4894-9526-ce0ba8ebc76a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c439aa23-c60e-413e-b940-d02c1d7b253f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c439aa23-c60e-413e-b940-d02c1d7b253f","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c439aa23-c60e-413e-b940-d02c1d7b253f","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434"}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:56:04.457Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","enabled":true}
{"level":"info","ts":"2026-07-10T10:56:04.469Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85916a7f-27f2-4894-9526-ce0ba8ebc76a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.469Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0c8c69b9-fe6f-42ec-9384-648b89d24c47","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.470Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0c8c69b9-fe6f-42ec-9384-648b89d24c47","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:56:04.470Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0c8c69b9-fe6f-42ec-9384-648b89d24c47","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:56:04.472Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3bb8e0c0-eed5-4fb5-a727-d30806d584a2","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:56:04.477Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T10:56:04.477Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:56:04.477Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:55:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-10T10:56:04.483Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3bb8e0c0-eed5-4fb5-a727-d30806d584a2","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:56:04.484Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3bb8e0c0-eed5-4fb5-a727-d30806d584a2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.484Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"41d8cf5a-90e9-463f-a73d-055dcbd01f11","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434"}
{"level":"error","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5194bf3c-2221-4256-9363-4a6d8ab21434","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:56:04.485Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c2684775-96f3-4f6a-b046-358beb05f934","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:55:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-10T10:56:04.525Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf"}
{"level":"info","ts":"2026-07-10T10:56:04.525Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf"}
{"level":"info","ts":"2026-07-10T10:56:04.525Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"84acae61-bedd-4a11-b68d-e8cf601dc029","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"84acae61-bedd-4a11-b68d-e8cf601dc029","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"84acae61-bedd-4a11-b68d-e8cf601dc029","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:56:04.526Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:55:57Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-10T10:56:04.530Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"41d8cf5a-90e9-463f-a73d-055dcbd01f11","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf"}
{"level":"error","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d4e2a136-4e47-482d-9284-8539d2e1b8cf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3"}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3"}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:56:04.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:04.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.534Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:04.534Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:04.534Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1af06d63-50b8-4c5b-97a6-dee75c0fb3c3","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:04.538Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084"}
{"level":"info","ts":"2026-07-10T10:56:04.538Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084"}
{"level":"info","ts":"2026-07-10T10:56:04.538Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:04.539Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"c84fe33d-1781-4e7b-a0cb-838ceff23084","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:04.541Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"41d8cf5a-90e9-463f-a73d-055dcbd01f11","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:56:04.541Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"41d8cf5a-90e9-463f-a73d-055dcbd01f11","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.541Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d184fd75-9f0e-4bb4-945b-a583cb08e3e4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d184fd75-9f0e-4bb4-945b-a583cb08e3e4","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:56:04.637Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d184fd75-9f0e-4bb4-945b-a583cb08e3e4","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:56:04.637Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d184fd75-9f0e-4bb4-945b-a583cb08e3e4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384"}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384"}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:04.928Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:04.929Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:04.929Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"30fa65d3-d7d5-4c92-ab1b-e45dd1e65384","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f"}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f"}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:06.005Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:06.006Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:06.006Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d58af850-ba3f-4a79-87db-899ca146453f","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:09.574Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.575Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"627d52ef-18be-40d8-b90b-c2a9ecc357e7","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"627d52ef-18be-40d8-b90b-c2a9ecc357e7","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"627d52ef-18be-40d8-b90b-c2a9ecc357e7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4cef2346-264e-4ff4-9f3a-758c51f7d60b","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cc40ce34-d306-4dec-b52f-0df383fb2328","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4cef2346-264e-4ff4-9f3a-758c51f7d60b","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4cef2346-264e-4ff4-9f3a-758c51f7d60b","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875"}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T10:56:09.586Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","enabled":true}
{"level":"info","ts":"2026-07-10T10:56:09.598Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cc40ce34-d306-4dec-b52f-0df383fb2328","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:56:09.599Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"627d52ef-18be-40d8-b90b-c2a9ecc357e7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.601Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","name":"ray-cluster-small-group-worker-xljkp"}
{"level":"info","ts":"2026-07-10T10:56:09.601Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:09.601Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:56:09.601Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:09Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875"}
{"level":"error","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8ce63dc1-9a17-49a3-9c70-42f699cf0875","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","podName":"ray-cluster-small-group-worker-xljkp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:56:09.609Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8cebf73e-2c49-405a-ad18-d5b261cb6256","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:09Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T10:56:09.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cc40ce34-d306-4dec-b52f-0df383fb2328","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:56:09.625Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cc40ce34-d306-4dec-b52f-0df383fb2328","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.625Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d067d95b-561e-4d4f-9e50-2ba58f58510f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.625Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4cef2346-264e-4ff4-9f3a-758c51f7d60b","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-d4rh6.svc","ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655"}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655"}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:56:09.626Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:04Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:09Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T10:56:09.633Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:56:09.633Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655"}
{"level":"error","ts":"2026-07-10T10:56:09.633Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9a238b14-5047-4274-b74b-8776ab21c655","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:56:09.638Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4cef2346-264e-4ff4-9f3a-758c51f7d60b","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-d4rh6.svc","ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-d4rh6.svc","*.ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*-worker-*.test-ns-d4rh6.svc","*-worker-*.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:56:09.638Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4cef2346-264e-4ff4-9f3a-758c51f7d60b","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T10:56:09.638Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cdad4f2d-c8d7-436b-96f0-8d31283a5c5f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.638Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cdad4f2d-c8d7-436b-96f0-8d31283a5c5f","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:56:09.638Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cdad4f2d-c8d7-436b-96f0-8d31283a5c5f","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:56:09.638Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"cdad4f2d-c8d7-436b-96f0-8d31283a5c5f","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c"}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c"}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:09.639Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6d752e34-ca83-45c3-b2e7-649c2c40873c","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d067d95b-561e-4d4f-9e50-2ba58f58510f","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48"}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48"}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:09.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:09.726Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:09.726Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8c7e8975-9bab-462d-8c06-6a998bbafe48","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:09.740Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d067d95b-561e-4d4f-9e50-2ba58f58510f","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:56:09.740Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d067d95b-561e-4d4f-9e50-2ba58f58510f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.740Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9ad49fc0-3b4d-486e-9f27-fe4fec93a492","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:09.754Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9ad49fc0-3b4d-486e-9f27-fe4fec93a492","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:56:09.767Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9ad49fc0-3b4d-486e-9f27-fe4fec93a492","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:56:09.767Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9ad49fc0-3b4d-486e-9f27-fe4fec93a492","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:10.058Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4"}
{"level":"info","ts":"2026-07-10T10:56:10.058Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4"}
{"level":"info","ts":"2026-07-10T10:56:10.058Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","podName":"ray-cluster-small-group-worker-xljkp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:10.059Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bf755216-7ba7-4ef8-ac15-63bb401a6aa4","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7"}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7"}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:10.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:56:10.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:10.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:56:10.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:56:10.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:56:10.319Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:56:10.319Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"75a70ba4-e7a1-4365-94dc-69cb908735c7","seconds":300}
{"level":"info","ts":"2026-07-10T10:56:36.848Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"011f0d9e-2dc3-476c-9e60-fec3bdaa2bfc","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:56:36.848Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"011f0d9e-2dc3-476c-9e60-fec3bdaa2bfc","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:56:36.848Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"011f0d9e-2dc3-476c-9e60-fec3bdaa2bfc","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:56:36.848Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"011f0d9e-2dc3-476c-9e60-fec3bdaa2bfc","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:56:36.863Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"011f0d9e-2dc3-476c-9e60-fec3bdaa2bfc","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-d4rh6.svc","ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.52","10.134.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:56:36.876Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"011f0d9e-2dc3-476c-9e60-fec3bdaa2bfc","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-d4rh6.svc","ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-d4rh6.svc","*.ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*-worker-*.test-ns-d4rh6.svc","*-worker-*.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.52","10.134.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:56:36.876Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"011f0d9e-2dc3-476c-9e60-fec3bdaa2bfc"}
{"level":"info","ts":"2026-07-10T10:57:06.877Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"23d74e3f-92ca-422a-86ee-ecdf1e299f3c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:06.877Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"23d74e3f-92ca-422a-86ee-ecdf1e299f3c","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:57:06.877Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"23d74e3f-92ca-422a-86ee-ecdf1e299f3c","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:57:06.877Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"23d74e3f-92ca-422a-86ee-ecdf1e299f3c","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:57:06.877Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"23d74e3f-92ca-422a-86ee-ecdf1e299f3c","rayCluster":"ray-cluster","podIPs":["10.133.0.52","10.134.0.18","10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64"}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64"}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:30.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:30.382Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:57:30.382Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:30.382Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"dfa5c860-3bcb-482b-a2fa-6af808dacc64","seconds":300}
{"level":"info","ts":"2026-07-10T10:57:31.391Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc"}
{"level":"info","ts":"2026-07-10T10:57:31.391Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc"}
{"level":"info","ts":"2026-07-10T10:57:31.391Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:31.392Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4e15c058-af57-4666-8dc6-007afbfbb1fc","seconds":300}
{"level":"info","ts":"2026-07-10T10:57:40.441Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2"}
{"level":"info","ts":"2026-07-10T10:57:40.441Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2"}
{"level":"info","ts":"2026-07-10T10:57:40.441Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:40.442Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"cca9f0fc-0465-4032-ae5a-e1f8667d2ee2","seconds":300}
{"level":"info","ts":"2026-07-10T10:57:40.644Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2"}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2"}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:40.645Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9dc27c1c-6ecb-4bab-ae0b-7a5b4204abc2","seconds":300}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d"}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d"}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:56:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:57:41.449Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"76c6ab87-62c2-4a4d-9c23-b12e7345871d","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:56:09Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:57:41Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T10:57:41.462Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4ddda474-bb2e-4ca3-8c17-90e13a0917d8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:41.462Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b83b6e81-cb91-456c-9368-1ba8e0521761","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:41.462Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4ddda474-bb2e-4ca3-8c17-90e13a0917d8","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:57:41.462Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4ddda474-bb2e-4ca3-8c17-90e13a0917d8","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:57:41.462Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4ddda474-bb2e-4ca3-8c17-90e13a0917d8","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:57:41.462Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4ddda474-bb2e-4ca3-8c17-90e13a0917d8","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52","10.134.0.18"]}
{"level":"info","ts":"2026-07-10T10:57:41.476Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b83b6e81-cb91-456c-9368-1ba8e0521761","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:57:41.487Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b83b6e81-cb91-456c-9368-1ba8e0521761","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:57:41.487Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b83b6e81-cb91-456c-9368-1ba8e0521761","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:41.646Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806"}
{"level":"info","ts":"2026-07-10T10:57:41.646Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806"}
{"level":"info","ts":"2026-07-10T10:57:41.646Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:41.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:41.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:41.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:41.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:41.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:41.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:41.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:41.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:41.647Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:56:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:57:41.647Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:41.647Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"edeb3cd9-c12a-44b3-a5cc-bf0b083a7806","seconds":300}
{"level":"info","ts":"2026-07-10T10:57:43.461Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4"}
{"level":"info","ts":"2026-07-10T10:57:43.461Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4"}
{"level":"info","ts":"2026-07-10T10:57:43.461Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","podName":"ray-cluster-small-group-worker-x2p7k","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:56:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:43.462Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6f3ff945-69e9-43a0-be8d-984d722f0ff4","seconds":300}
{"level":"info","ts":"2026-07-10T10:57:51.689Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e"}
{"level":"info","ts":"2026-07-10T10:57:51.689Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e"}
{"level":"info","ts":"2026-07-10T10:57:51.689Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","podName":"ray-cluster-small-group-worker-xljkp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:51.690Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"010db9c5-aca6-4cea-85a0-e115ce46f39e","seconds":300}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d"}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d"}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","podName":"ray-cluster-small-group-worker-xljkp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:57:52.472Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ddbb1dcd-5474-4713-9402-fb410266c73d","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:57:41Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:57:52Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T10:57:52.485Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"bc93f46a-af49-4177-9e4b-f03197a9d33a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:52.485Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e57746f5-7118-459c-bc0d-8ba1e08c9a50","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:52.485Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"bc93f46a-af49-4177-9e4b-f03197a9d33a","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:57:52.485Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"bc93f46a-af49-4177-9e4b-f03197a9d33a","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:57:52.485Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"bc93f46a-af49-4177-9e4b-f03197a9d33a","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:57:52.485Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"bc93f46a-af49-4177-9e4b-f03197a9d33a","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52","10.134.0.18"]}
{"level":"info","ts":"2026-07-10T10:57:52.496Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e57746f5-7118-459c-bc0d-8ba1e08c9a50","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:57:52.505Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e57746f5-7118-459c-bc0d-8ba1e08c9a50","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:57:52.505Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e57746f5-7118-459c-bc0d-8ba1e08c9a50","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815"}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815"}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:52.694Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","podName":"ray-cluster-small-group-worker-xljkp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:56:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:57:52.695Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:57:52.695Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d8fce863-3b62-46bf-af7a-b764375ef815","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:57:52Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:57:52Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T10:57:52.707Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"220d3211-0112-4793-a1d0-1bb616f37fef","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:52.707Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a6564caf-fe21-4d9a-af48-8bbe0fcade8f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:52.707Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a6564caf-fe21-4d9a-af48-8bbe0fcade8f","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:57:52.707Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a6564caf-fe21-4d9a-af48-8bbe0fcade8f","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:57:52.707Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a6564caf-fe21-4d9a-af48-8bbe0fcade8f","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:57:52.708Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a6564caf-fe21-4d9a-af48-8bbe0fcade8f","rayCluster":"ray-cluster","podIPs":["10.133.0.52","10.134.0.18","10.132.0.36"]}
{"level":"info","ts":"2026-07-10T10:57:52.719Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"220d3211-0112-4793-a1d0-1bb616f37fef","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:57:52.729Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"220d3211-0112-4793-a1d0-1bb616f37fef","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:57:52.729Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"220d3211-0112-4793-a1d0-1bb616f37fef","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed"}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed"}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","podName":"ray-cluster-small-group-worker-xljkp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T10:56:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:57:54.486Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"da935f51-5f63-484a-9bec-d5a9d3c141ed","seconds":300}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de"}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de"}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:58:03.724Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"ce6f0b22-87ad-4d66-a900-0d4596ab07de","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:57:52Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:58:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T10:58:03.738Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8de26f67-9873-480c-b4f0-9ffad2b552c6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:58:03.738Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0f9033e6-5760-4eff-90a7-943909a52bec","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:58:03.738Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0f9033e6-5760-4eff-90a7-943909a52bec","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:58:03.738Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0f9033e6-5760-4eff-90a7-943909a52bec","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:58:03.738Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0f9033e6-5760-4eff-90a7-943909a52bec","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:58:03.738Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0f9033e6-5760-4eff-90a7-943909a52bec","rayCluster":"ray-cluster","podIPs":["10.134.0.18","10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T10:58:03.757Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8de26f67-9873-480c-b4f0-9ffad2b552c6","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:58:03.769Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8de26f67-9873-480c-b4f0-9ffad2b552c6","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:58:03.769Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8de26f67-9873-480c-b4f0-9ffad2b552c6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1"}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1"}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T10:58:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:58:05.739Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:58:05.739Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"5ae16f72-f095-4c06-9900-f6b0f7a5d4b1","seconds":300}
{"level":"info","ts":"2026-07-10T10:58:06.878Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4aa0fe0c-1af4-47e6-a280-9d5246081d96","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:58:06.878Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4aa0fe0c-1af4-47e6-a280-9d5246081d96","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:58:06.878Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4aa0fe0c-1af4-47e6-a280-9d5246081d96","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:58:06.878Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4aa0fe0c-1af4-47e6-a280-9d5246081d96","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:58:06.878Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4aa0fe0c-1af4-47e6-a280-9d5246081d96","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52","10.134.0.18"]}
{"level":"info","ts":"2026-07-10T10:59:01.974Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:01.975Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5dbd85fc-35df-4efb-ae0b-017afcc3854a","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"f4485142-5217-4031-9963-107f26751051","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5dbd85fc-35df-4efb-ae0b-017afcc3854a","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5dbd85fc-35df-4efb-ae0b-017afcc3854a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"605b9f09-96d4-42f8-a5af-7e0a27cb2caa","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"605b9f09-96d4-42f8-a5af-7e0a27cb2caa","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"605b9f09-96d4-42f8-a5af-7e0a27cb2caa","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"605b9f09-96d4-42f8-a5af-7e0a27cb2caa","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","worker Pod":"ray-cluster-small-group-worker-xljkp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xljkp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-xljkp"}
{"level":"info","ts":"2026-07-10T10:59:01.986Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"605b9f09-96d4-42f8-a5af-7e0a27cb2caa","rayCluster":"ray-cluster","podIPs":["10.134.0.18","10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T10:59:01.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:59:01.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:59:01.996Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:59:01.996Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:58:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:01Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T10:59:01.998Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5dbd85fc-35df-4efb-ae0b-017afcc3854a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:01.998Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9a29c403-1be0-4b8a-b6e5-e8d4c5a970f0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:01.998Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9a29c403-1be0-4b8a-b6e5-e8d4c5a970f0","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:59:01.998Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9a29c403-1be0-4b8a-b6e5-e8d4c5a970f0","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:59:01.998Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9a29c403-1be0-4b8a-b6e5-e8d4c5a970f0","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:59:01.998Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9a29c403-1be0-4b8a-b6e5-e8d4c5a970f0","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52","10.134.0.18"]}
{"level":"info","ts":"2026-07-10T10:59:02.004Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"f4485142-5217-4031-9963-107f26751051","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:59:02.004Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:59:02.004Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115"}
{"level":"error","ts":"2026-07-10T10:59:02.004Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e9a2f8d3-c424-4da6-8533-f99448dd8115","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:59:02.004Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab"}
{"level":"info","ts":"2026-07-10T10:59:02.004Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab"}
{"level":"info","ts":"2026-07-10T10:59:02.004Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:02.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:02.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:02.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:59:02.126Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:59:02.126Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6e129020-4b47-4fec-830d-e2327cf59aab","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:58:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:02Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T10:59:02.225Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"f4485142-5217-4031-9963-107f26751051","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:59:02.225Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"f4485142-5217-4031-9963-107f26751051","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.225Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031e9e67-5b09-4328-bea8-82d5897bdb36","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6e55a02e-59b2-4753-aba7-b0647b917380","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6e55a02e-59b2-4753-aba7-b0647b917380","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6e55a02e-59b2-4753-aba7-b0647b917380","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:59:02.226Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6e55a02e-59b2-4753-aba7-b0647b917380","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:59:02.227Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:59:02.227Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:58:03Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:02Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T10:59:02.235Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T10:59:02.235Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f"}
{"level":"error","ts":"2026-07-10T10:59:02.235Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"34d19ba3-bb12-4f46-8dc4-39728e6a327f","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T10:59:02.239Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031e9e67-5b09-4328-bea8-82d5897bdb36","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:59:02.241Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a"}
{"level":"info","ts":"2026-07-10T10:59:02.241Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a"}
{"level":"info","ts":"2026-07-10T10:59:02.241Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:02.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:02.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:02.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:59:02.325Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"6e55a02e-59b2-4753-aba7-b0647b917380","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52","10.134.0.18"]}
{"level":"info","ts":"2026-07-10T10:59:02.325Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:59:02.325Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"920f2bd9-3d10-4f19-ab2d-a7fc9055e12a","seconds":300}
{"level":"info","ts":"2026-07-10T10:59:02.330Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031e9e67-5b09-4328-bea8-82d5897bdb36","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:59:02.330Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031e9e67-5b09-4328-bea8-82d5897bdb36","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.330Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031d5e0d-e5de-4095-9657-88da96391cda","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031d5e0d-e5de-4095-9657-88da96391cda","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:59:02.351Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031d5e0d-e5de-4095-9657-88da96391cda","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:59:02.351Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"031d5e0d-e5de-4095-9657-88da96391cda","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.575Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5"}
{"level":"info","ts":"2026-07-10T10:59:02.575Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5"}
{"level":"info","ts":"2026-07-10T10:59:02.575Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:02.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:02.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:02.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:59:02.576Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T10:59:02.576Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"e6f5f253-c77c-4b3f-ac8d-a8f41d50afb5","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:02Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:02Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T10:59:02.588Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b1ba0c5a-b6ea-4ef0-b71b-ef63631fc7f9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.588Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"661f7cf3-8213-41a8-882d-8061ac637f63","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.588Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b1ba0c5a-b6ea-4ef0-b71b-ef63631fc7f9","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:59:02.588Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b1ba0c5a-b6ea-4ef0-b71b-ef63631fc7f9","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:59:02.588Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b1ba0c5a-b6ea-4ef0-b71b-ef63631fc7f9","pod":"ray-cluster-small-group-worker-xljkp","ip":"10.134.0.18"}
{"level":"info","ts":"2026-07-10T10:59:02.588Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b1ba0c5a-b6ea-4ef0-b71b-ef63631fc7f9","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52","10.134.0.18"]}
{"level":"info","ts":"2026-07-10T10:59:02.598Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"661f7cf3-8213-41a8-882d-8061ac637f63","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T10:59:02.610Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"661f7cf3-8213-41a8-882d-8061ac637f63","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T10:59:02.610Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"661f7cf3-8213-41a8-882d-8061ac637f63","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9"}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9"}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:59:02.940Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"019c437c-ff44-4dc4-af9d-65bcc1f8bae9","seconds":300}
{"level":"info","ts":"2026-07-10T10:59:02.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282"}
{"level":"info","ts":"2026-07-10T10:59:02.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282"}
{"level":"info","ts":"2026-07-10T10:59:02.944Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:02.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:02.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:02.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:59:02.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:02.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:59:02.944Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-xljkp"}
{"level":"info","ts":"2026-07-10T10:59:02.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","The worker Pod has already been deleted":"ray-cluster-small-group-worker-xljkp"}
{"level":"info","ts":"2026-07-10T10:59:02.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:59:02.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:59:02.947Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:59:02.947Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8796045c-56c4-4809-9002-fa70a5629282","seconds":300}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50"}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50"}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T10:59:04.588Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-xljkp"}
{"level":"info","ts":"2026-07-10T10:59:04.592Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","The worker Pod has already been deleted":"ray-cluster-small-group-worker-xljkp"}
{"level":"info","ts":"2026-07-10T10:59:04.592Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T10:59:04.592Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T10:59:04.592Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T10:59:04.592Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"addbcb36-10c2-4ee2-a12a-1caba1687c50","seconds":300}
{"level":"info","ts":"2026-07-10T10:59:06.879Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c2e0a689-ac09-46bf-86a7-eeb536ee060c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T10:59:06.879Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c2e0a689-ac09-46bf-86a7-eeb536ee060c","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T10:59:06.879Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c2e0a689-ac09-46bf-86a7-eeb536ee060c","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T10:59:06.906Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c2e0a689-ac09-46bf-86a7-eeb536ee060c","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-d4rh6.svc","ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:59:06.918Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c2e0a689-ac09-46bf-86a7-eeb536ee060c","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-d4rh6.svc","ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-d4rh6.svc","*.ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*-worker-*.test-ns-d4rh6.svc","*-worker-*.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","10.133.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T10:59:06.918Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c2e0a689-ac09-46bf-86a7-eeb536ee060c","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T11:00:06.919Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"166f0b34-4904-424e-9c99-034e5e8d2f90","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:06.919Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"166f0b34-4904-424e-9c99-034e5e8d2f90","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T11:00:06.919Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"166f0b34-4904-424e-9c99-034e5e8d2f90","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T11:00:06.919Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"166f0b34-4904-424e-9c99-034e5e8d2f90","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T11:00:12.981Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:12.982Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"653c009e-8cce-4305-9fbe-3563f4da3b72","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"653c009e-8cce-4305-9fbe-3563f4da3b72","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"653c009e-8cce-4305-9fbe-3563f4da3b72","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"257befbb-2ae6-49a3-9483-9d1f24983f6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a1e0207d-882a-4952-a203-1d2d24cd8d04","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a1e0207d-882a-4952-a203-1d2d24cd8d04","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a1e0207d-882a-4952-a203-1d2d24cd8d04","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a1e0207d-882a-4952-a203-1d2d24cd8d04","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","worker Pod":"ray-cluster-small-group-worker-x2p7k","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-x2p7k. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:12.992Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:13.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:13.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:13.002Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:00:13.002Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:02Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:13Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:00:13.009Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"653c009e-8cce-4305-9fbe-3563f4da3b72","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.010Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0a403653-75d3-4cca-8bcd-51e4622f677a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.010Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0a403653-75d3-4cca-8bcd-51e4622f677a","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T11:00:13.010Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0a403653-75d3-4cca-8bcd-51e4622f677a","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T11:00:13.010Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"0a403653-75d3-4cca-8bcd-51e4622f677a","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T11:00:13.011Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"257befbb-2ae6-49a3-9483-9d1f24983f6e","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027"}
{"level":"error","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"9e90e4b4-344f-4ce3-a9a4-ae6f8c738027","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628"}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628"}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:00:13.015Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"163bc0da-4561-48b7-a259-b0e6f7dcd628","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:02Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:13Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:00:13.022Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"257befbb-2ae6-49a3-9483-9d1f24983f6e","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:13.022Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"257befbb-2ae6-49a3-9483-9d1f24983f6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.022Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"45d56d26-3fe9-4455-aac7-5037ee1a46c1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"173efcae-9950-47f4-945d-ff20ffc2d060","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"173efcae-9950-47f4-945d-ff20ffc2d060","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T10:59:02Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:13Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:00:13.026Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"173efcae-9950-47f4-945d-ff20ffc2d060","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T11:00:13.027Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"173efcae-9950-47f4-945d-ff20ffc2d060","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T11:00:13.032Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-d4rh6","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:00:13.032Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77"}
{"level":"error","ts":"2026-07-10T11:00:13.032Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"365104fb-548a-4d49-af78-cef6b984ed77","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:13.032Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca"}
{"level":"info","ts":"2026-07-10T11:00:13.032Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca"}
{"level":"info","ts":"2026-07-10T11:00:13.032Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:13.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:13.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:13.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:00:13.033Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:13.033Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"b5309ddd-bc21-4b6d-b400-3012f9331fca","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:13.035Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"45d56d26-3fe9-4455-aac7-5037ee1a46c1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40"}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40"}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:13.038Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f830050f-0026-43e8-919b-302cfad49a40","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:13.043Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"45d56d26-3fe9-4455-aac7-5037ee1a46c1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:13.043Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"45d56d26-3fe9-4455-aac7-5037ee1a46c1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.043Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5c8b432e-5137-469f-aea6-ca1f9be23f6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.052Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5c8b432e-5137-469f-aea6-ca1f9be23f6e","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:13.130Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5c8b432e-5137-469f-aea6-ca1f9be23f6e","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:13.130Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"5c8b432e-5137-469f-aea6-ca1f9be23f6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2"}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2"}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:00:13.470Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"bbcc9d87-e1a8-482d-a888-6f7500c7e1b2","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:13Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:13Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:00:13.481Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2b048b15-d6e5-46b3-a103-ff6e7691ddd0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.481Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1de50085-d84d-4d93-8f31-2a180fe1b93c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.481Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1de50085-d84d-4d93-8f31-2a180fe1b93c","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T11:00:13.481Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1de50085-d84d-4d93-8f31-2a180fe1b93c","pod":"ray-cluster-small-group-worker-x2p7k","ip":"10.133.0.52"}
{"level":"info","ts":"2026-07-10T11:00:13.482Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1de50085-d84d-4d93-8f31-2a180fe1b93c","rayCluster":"ray-cluster","podIPs":["10.132.0.36","10.133.0.52"]}
{"level":"info","ts":"2026-07-10T11:00:13.491Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2b048b15-d6e5-46b3-a103-ff6e7691ddd0","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:13.503Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2b048b15-d6e5-46b3-a103-ff6e7691ddd0","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:13.503Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2b048b15-d6e5-46b3-a103-ff6e7691ddd0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.718Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.718Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pxgtj"}
{"level":"info","ts":"2026-07-10T11:00:13.728Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:00:13.728Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.728Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.728Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f116bf7a-126c-4f8c-8011-8965da74143a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.728Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.728Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.728Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.733Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.733Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pxgtj"}
{"level":"info","ts":"2026-07-10T11:00:13.735Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.741Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.741Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.747Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f116bf7a-126c-4f8c-8011-8965da74143a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.747Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5b5276e1-cb23-4d94-91f8-87a6b9d8b0f1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.753Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster","secret":"ca-secret"}
{"level":"info","ts":"2026-07-10T11:00:13.754Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.754Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:13.754Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.758Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5b5276e1-cb23-4d94-91f8-87a6b9d8b0f1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:13.766Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:13.766Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:285\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:13.766Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"acccdbe5-5c84-4dcb-ad1f-21d2875048ef","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.766Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0b43e087-f144-4106-9a56-4d93a638f745","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:00:13.766Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0b43e087-f144-4106-9a56-4d93a638f745","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.766Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0b43e087-f144-4106-9a56-4d93a638f745","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.766Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0b43e087-f144-4106-9a56-4d93a638f745","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.772Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5b5276e1-cb23-4d94-91f8-87a6b9d8b0f1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:13.772Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5b5276e1-cb23-4d94-91f8-87a6b9d8b0f1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.772Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"1c68a416-a5b8-4fed-8a69-f25a2edbb5be","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.825Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-10T11:00:13.825Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0b43e087-f144-4106-9a56-4d93a638f745","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.829Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"1c68a416-a5b8-4fed-8a69-f25a2edbb5be","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:13.833Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-10T11:00:13.833Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b53a5a77-3ff2-497a-b3a9-9cd35414cad7"}
{"level":"info","ts":"2026-07-10T11:00:13.833Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f78331c1-b76d-4527-a9ac-ae04862bff47","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.833Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f78331c1-b76d-4527-a9ac-ae04862bff47"}
{"level":"info","ts":"2026-07-10T11:00:13.840Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"1c68a416-a5b8-4fed-8a69-f25a2edbb5be","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:13.840Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"1c68a416-a5b8-4fed-8a69-f25a2edbb5be","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.840Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"90f3d78d-5fb8-4c16-bd1c-6ae9c09bb3e8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.851Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"90f3d78d-5fb8-4c16-bd1c-6ae9c09bb3e8","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:13.925Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"90f3d78d-5fb8-4c16-bd1c-6ae9c09bb3e8","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:13.925Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"90f3d78d-5fb8-4c16-bd1c-6ae9c09bb3e8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:13.925Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.077Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.077Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83"}
{"level":"info","ts":"2026-07-10T11:00:14.077Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83"}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:00:14.085Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83"}
{"level":"error","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7c4523-eb7d-41ab-b474-0b531390ab83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a"}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a"}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:00:14.092Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:13Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:13Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a"}
{"level":"error","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9b52d07d-88c2-4886-9952-7012327d932a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18"}
{"level":"info","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18"}
{"level":"info","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.102Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:13Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:13Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a4f3d51b-7658-42af-94ab-0b179570e25c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"609d98b4-219e-4292-a6af-fae6ffd43740","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.103Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"609d98b4-219e-4292-a6af-fae6ffd43740"}
{"level":"info","ts":"2026-07-10T11:00:14.108Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:00:14.108Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18"}
{"level":"error","ts":"2026-07-10T11:00:14.108Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6b721a52-3915-46a9-b781-924c3fff7e18","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349"}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349"}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.114Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349"}
{"level":"error","ts":"2026-07-10T11:00:14.114Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"add62592-61a5-4c81-9de3-60f749d26349","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.117Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a4f3d51b-7658-42af-94ab-0b179570e25c","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694"}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694"}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:14.123Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a5b6b9c0-18bb-4048-a678-303b66e7c694","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:14.128Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a4f3d51b-7658-42af-94ab-0b179570e25c","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:14.128Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a4f3d51b-7658-42af-94ab-0b179570e25c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7"}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7"}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:14.129Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:14.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","The worker Pod has already been deleted":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:14.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:14.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:14.132Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:14.132Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0104cfe3-dd09-4de9-bd52-54b8aa8a2fa7","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:14.154Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a"}
{"level":"info","ts":"2026-07-10T11:00:14.154Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a"}
{"level":"info","ts":"2026-07-10T11:00:14.154Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a"}
{"level":"error","ts":"2026-07-10T11:00:14.155Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cb42de19-a77e-407f-807f-622df9bdeb3a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.175Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","roleBinding":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:14.183Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","name":"kuberay-gateway-access","namespace":"test-ns-pxgtj","operation":"created"}
{"level":"info","ts":"2026-07-10T11:00:14.190Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","name":"test-ns-pxgtj-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-07-10T11:00:14.195Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"error","ts":"2026-07-10T11:00:14.201Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.201Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3"}
{"level":"error","ts":"2026-07-10T11:00:14.201Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"afceeca9-0703-46a7-8301-d8e8321c6ad3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.206Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"3fb0644a-e2b7-4d36-8793-74afb52c2eb7","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:00:14.206Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"3fb0644a-e2b7-4d36-8793-74afb52c2eb7","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.206Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"3fb0644a-e2b7-4d36-8793-74afb52c2eb7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.216Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"3fb0644a-e2b7-4d36-8793-74afb52c2eb7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.217Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"7659450a-6e92-49ac-bf60-fe63a88bfc1a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.217Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"d4cbe77e-6b94-4155-8f87-596f583c40d0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.217Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"d4cbe77e-6b94-4155-8f87-596f583c40d0"}
{"level":"info","ts":"2026-07-10T11:00:14.229Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"7659450a-6e92-49ac-bf60-fe63a88bfc1a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:14.235Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716"}
{"level":"info","ts":"2026-07-10T11:00:14.235Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716"}
{"level":"info","ts":"2026-07-10T11:00:14.235Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.235Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.235Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.236Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.236Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.236Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.236Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.236Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716"}
{"level":"error","ts":"2026-07-10T11:00:14.236Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e5e729f6-96fa-4728-b356-8370f509a716","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.239Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"7659450a-6e92-49ac-bf60-fe63a88bfc1a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:14.239Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"7659450a-6e92-49ac-bf60-fe63a88bfc1a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6"}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6"}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6"}
{"level":"error","ts":"2026-07-10T11:00:14.397Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"13b54fa2-5eba-4ecc-aed4-426e8c2978a6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.717Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4"}
{"level":"info","ts":"2026-07-10T11:00:14.717Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4"}
{"level":"info","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4"}
{"level":"error","ts":"2026-07-10T11:00:14.718Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa1c0671-3101-4eab-8aa5-9b77f3ad6ba4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:15.359Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","enabled":true}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","name":"ray-cluster-head-wvb8f"}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","filter labels":["test-ns-pxgtj",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:15.378Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e25be4ec-a5ea-4b82-879b-7c038551679d","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f"}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f"}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.379Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.380Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"ray-cluster-head-svc"},"newHead":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-10T11:00:15.380Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e01df24-2c0f-4c47-bdc2-1d0d797a070f","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:00:15.391Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"320b74c0-5e43-4e7f-b2cf-aea435b86399","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6f1e061a-e567-46ef-b955-1a5b8a59b0a0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"320b74c0-5e43-4e7f-b2cf-aea435b86399"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"ray-cluster-head-svc"},"newHead":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-10T11:00:15.392Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf"}
{"level":"error","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1bc5a9c3-34a1-4ad8-8f23-f33728ad62cf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.401Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.402Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:15.402Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"11bfcd5b-215e-4cc6-b5fb-e2899648ab89","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6f1e061a-e567-46ef-b955-1a5b8a59b0a0","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:15.407Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"dafb7a35-e871-45e1-b83a-0b1be6631c10","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:15.418Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6f1e061a-e567-46ef-b955-1a5b8a59b0a0","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:15.418Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6f1e061a-e567-46ef-b955-1a5b8a59b0a0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-10T11:00:15.426Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"22dab77b-c9ee-4be3-ab63-f131af906160","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:00:15.437Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"032fab8f-4503-469c-b268-3d9038b0a753","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:15.437Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f81c7502-a4b5-4a16-96c6-3a8fc4ac0f32","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:15.437Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f81c7502-a4b5-4a16-96c6-3a8fc4ac0f32"}
{"level":"info","ts":"2026-07-10T11:00:15.446Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"032fab8f-4503-469c-b268-3d9038b0a753","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:15.459Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"032fab8f-4503-469c-b268-3d9038b0a753","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:15.459Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"032fab8f-4503-469c-b268-3d9038b0a753","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d"}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d"}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.482Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:15.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","The worker Pod has already been deleted":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:15.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.487Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:15.487Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"8036f7db-d582-4793-8a3b-56a73033595d","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:15.832Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c"}
{"level":"info","ts":"2026-07-10T11:00:15.832Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c"}
{"level":"info","ts":"2026-07-10T11:00:15.832Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:15.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:15.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:15.833Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1a703fd4-a2da-48fe-8d2a-ed3d377add7c","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"newHead":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-10T11:00:16.139Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"020bf883-2453-4c84-91d9-35b5b5290f04","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:00:16.151Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0dcfd607-d079-44a4-a1ff-08490d3b6c50","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:16.151Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"be02a935-acf4-4510-929d-5e9b676a2aa0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:16.151Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0dcfd607-d079-44a4-a1ff-08490d3b6c50","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:00:16.163Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"be02a935-acf4-4510-929d-5e9b676a2aa0","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:00:16.164Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0dcfd607-d079-44a4-a1ff-08490d3b6c50","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pxgtj.svc","ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:00:16.175Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"be02a935-acf4-4510-929d-5e9b676a2aa0","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:00:16.175Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"be02a935-acf4-4510-929d-5e9b676a2aa0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:16.176Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0dcfd607-d079-44a4-a1ff-08490d3b6c50","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pxgtj.svc","ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pxgtj.svc","*.ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*-worker-*.test-ns-pxgtj.svc","*-worker-*.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:00:16.176Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0dcfd607-d079-44a4-a1ff-08490d3b6c50","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:17.437Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"48e7204a-aafa-40d3-8399-40130fa4c441","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b"}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b"}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:18.722Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","The worker Pod has already been deleted":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:18.726Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:18.726Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"4532bb29-cebb-443d-88a4-fb6026e71e8b","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a"}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a"}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","head Pod":"ray-cluster-head-b4ww4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-b4ww4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:18.746Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:18.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","The worker Pod has already been deleted":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:18.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:18.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:18.749Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:18.749Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"03dbeaa9-315a-47c9-8af9-e7953e412a9a","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:43.834Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"d97e541b-93d5-4265-a849-58d6fa9052f0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:43.834Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"d97e541b-93d5-4265-a849-58d6fa9052f0","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:00:43.834Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"d97e541b-93d5-4265-a849-58d6fa9052f0","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:00:48.936Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c"}
{"level":"info","ts":"2026-07-10T11:00:48.936Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c"}
{"level":"info","ts":"2026-07-10T11:00:48.937Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:48.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","Found 1 head Pod":"ray-cluster-head-b4ww4","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-10 10:55:35 +0000 UTC,FinishedAt:2026-07-10 11:00:21 +0000 UTC,ContainerID:cri-o://283dd07314c2aa33f20202769e132b3594db4d62888cfe3c375c32288e4f9cda,}"}
{"level":"info","ts":"2026-07-10T11:00:48.937Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","head Pod":"ray-cluster-head-b4ww4","shouldDelete":true,"reason":"The head Pod ray-cluster-head-b4ww4 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-10T11:00:48.942Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod ray-cluster-head-b4ww4 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:48.942Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","error":"The head Pod ray-cluster-head-b4ww4 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-10T11:00:48.942Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:48Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-10T11:00:48.942Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:13Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:48Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:00:48.953Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c"}
{"level":"error","ts":"2026-07-10T11:00:48.953Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0cd5b9b9-ce4c-4f5a-920a-cc046825954c","error":"The head Pod ray-cluster-head-b4ww4 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:48.953Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"396482fd-adfa-4d5b-8da0-46971f98e3df","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:48.953Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a78feca9-100c-4e53-b8a2-636415677ac6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:48.953Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"396482fd-adfa-4d5b-8da0-46971f98e3df","pod":"ray-cluster-head-b4ww4","ip":"10.132.0.36"}
{"level":"info","ts":"2026-07-10T11:00:48.958Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3"}
{"level":"info","ts":"2026-07-10T11:00:48.958Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3"}
{"level":"info","ts":"2026-07-10T11:00:48.958Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:48.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T11:00:48.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:48.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:48.958Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:48.959Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a78feca9-100c-4e53-b8a2-636415677ac6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","The worker Pod has already been deleted":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:48.961Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:48.961Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T11:00:48.962Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:48.962Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"02d86db2-f953-4039-b702-2af5acd5e0f3","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:48.965Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"396482fd-adfa-4d5b-8da0-46971f98e3df","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-d4rh6.svc","ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:00:48.978Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"396482fd-adfa-4d5b-8da0-46971f98e3df","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-d4rh6.svc","ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-d4rh6.svc","*.ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*-worker-*.test-ns-d4rh6.svc","*-worker-*.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["10.132.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:00:48.978Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"396482fd-adfa-4d5b-8da0-46971f98e3df","rayCluster":"ray-cluster","podIPs":["10.132.0.36"]}
{"level":"info","ts":"2026-07-10T11:00:49.133Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af"}
{"level":"info","ts":"2026-07-10T11:00:49.133Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af"}
{"level":"info","ts":"2026-07-10T11:00:49.133Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-10T11:00:49.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:00:49.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:49.133Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","namespace":"test-ns-d4rh6","name":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:49.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","The worker Pod has already been deleted":"ray-cluster-small-group-worker-x2p7k"}
{"level":"info","ts":"2026-07-10T11:00:49.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:00:49.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:00:49.136Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","podName":"ray-cluster-head-b4ww4","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-10T11:00:49.137Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:00:49.137Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"f03a29e4-8578-4581-bbcc-6676bd0ed0af","seconds":300}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.138Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.139Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.139Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.139Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.142Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.142Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.142Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.142Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.142Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.142Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"newHead":{"serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-10T11:00:49.142Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:48Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"ray-cluster-head-b4ww4","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:48Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:49Z","stateTransitionTimes":{"ready":"2026-07-10T10:55:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T10:54:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:48Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T10:55:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T10:54:07Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-10T11:00:49Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:00:49.153Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f"}
{"level":"error","ts":"2026-07-10T11:00:49.153Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"810989db-81cb-4efb-bf64-0974f6394e5f","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.153Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"7097aec9-d0e1-478d-9f5d-9ac27a99d007","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.153Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a08cb036-996c-4189-8f8f-0428c125b1d6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.158Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a08cb036-996c-4189-8f8f-0428c125b1d6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.159Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.162Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.162Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.162Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.162Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.162Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.162Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c"}
{"level":"error","ts":"2026-07-10T11:00:49.162Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"05f673c0-c800-43e9-b1a7-68529c14923c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.166Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"7097aec9-d0e1-478d-9f5d-9ac27a99d007","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-d4rh6.svc","ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.173Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.225Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.225Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"7097aec9-d0e1-478d-9f5d-9ac27a99d007","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-d4rh6.svc","ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-d4rh6.svc","*.ray-cluster-worker-svc.test-ns-d4rh6.svc.cluster.local","*-worker-*.test-ns-d4rh6.svc","*-worker-*.test-ns-d4rh6.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:00:49.225Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"7097aec9-d0e1-478d-9f5d-9ac27a99d007"}
{"level":"info","ts":"2026-07-10T11:00:49.225Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.225Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.225Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac"}
{"level":"error","ts":"2026-07-10T11:00:49.225Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"d7b4b74c-c321-4286-ac17-26925f5b1cac","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.246Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.250Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.250Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.250Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2"}
{"level":"error","ts":"2026-07-10T11:00:49.250Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"0fa20e89-5e64-4828-924b-4bbe35e9b3c2","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.291Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.296Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.296Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.296Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.296Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a"}
{"level":"error","ts":"2026-07-10T11:00:49.296Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"af8b2630-c111-48a8-ae87-0d0e41452b4a","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.377Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.381Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.381Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.381Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.382Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5"}
{"level":"error","ts":"2026-07-10T11:00:49.382Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"070747de-e2f4-4377-9756-7f3ab46019e5","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.542Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.546Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.546Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.546Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603"}
{"level":"error","ts":"2026-07-10T11:00:49.546Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"1d8d6733-974b-47c9-b7f6-bab551ab4603","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:49.868Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:49.872Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:49.872Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:49.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:49.872Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc"}
{"level":"error","ts":"2026-07-10T11:00:49.872Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"91e2cc82-8196-43bf-bed5-663ccc2174fc","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:50.513Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69"}
{"level":"info","ts":"2026-07-10T11:00:50.513Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69"}
{"level":"info","ts":"2026-07-10T11:00:50.513Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:50.513Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69"}
{"level":"info","ts":"2026-07-10T11:00:50.513Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69"}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:50.514Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:50.518Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:50.518Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:50.518Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:50.518Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:50.518Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:50.518Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69"}
{"level":"error","ts":"2026-07-10T11:00:50.518Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"024418d1-ddde-421d-bbe5-07532fbabc69","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:51.799Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:51.803Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:51.803Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:51.803Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:51.803Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:51.803Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:51.803Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb"}
{"level":"error","ts":"2026-07-10T11:00:51.803Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"13f17d7b-92b3-47a6-a221-f5eed4cb13cb","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-10T11:00:54.364Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-10T11:00:54.365Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:54.365Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-d4rh6.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-10T11:00:54.365Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","enabled":true}
{"level":"error","ts":"2026-07-10T11:00:54.368Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:54.368Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:54.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:54.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:54.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:54.369Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832"}
{"level":"error","ts":"2026-07-10T11:00:54.369Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"6aa63630-53b1-4ef9-9564-50a6f94c2832","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.574Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"134a1bb8-a00d-455e-9a4a-4f866a70579a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.579Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"134a1bb8-a00d-455e-9a4a-4f866a70579a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.579Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ecca89cf-693b-499a-931b-108a86a2dc3e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.584Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ecca89cf-693b-499a-931b-108a86a2dc3e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.770Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d9f17cf4-b65e-439b-9f51-0c58aa7115cb","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:56.770Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d9f17cf4-b65e-439b-9f51-0c58aa7115cb","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.770Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d9f17cf4-b65e-439b-9f51-0c58aa7115cb","cluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileAutoscalerRole-fm","error":"roles.rbac.authorization.k8s.io \"ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3","error":"roles.rbac.authorization.k8s.io \"ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated"}
{"level":"info","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3","serviceSelectors":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/identifier":"ray-cluster-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3","filter labels":["test-ns-d4rh6",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3","error":"unable to find head service. cluster name ray-cluster, filter labels [test-ns-d4rh6 map[ray.io/cluster:ray-cluster ray.io/identifier:ray-cluster-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3"}
{"level":"error","ts":"2026-07-10T11:00:56.772Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"168e6d6f-b0ca-4435-a058-497ad152dfb3","error":"roles.rbac.authorization.k8s.io \"ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.778Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d9f17cf4-b65e-439b-9f51-0c58aa7115cb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.931Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d46a8412-e8db-4361-9181-4dd1e99d8c00","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:56.931Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d46a8412-e8db-4361-9181-4dd1e99d8c00","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.931Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d46a8412-e8db-4361-9181-4dd1e99d8c00","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.942Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"d46a8412-e8db-4361-9181-4dd1e99d8c00","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.942Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85a76bd4-aa4c-4115-ae7f-0c291a0f136d","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:56.942Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85a76bd4-aa4c-4115-ae7f-0c291a0f136d","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.942Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85a76bd4-aa4c-4115-ae7f-0c291a0f136d","cluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:56.956Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85a76bd4-aa4c-4115-ae7f-0c291a0f136d","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.956Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85a76bd4-aa4c-4115-ae7f-0c291a0f136d"}
{"level":"error","ts":"2026-07-10T11:00:56.956Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"85a76bd4-aa4c-4115-ae7f-0c291a0f136d","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.956Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2cef478f-c587-44c1-b2cf-ada768dcee45","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.956Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e45e6816-cd66-4fb0-9bae-442daa0cb7bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.956Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2cef478f-c587-44c1-b2cf-ada768dcee45","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:56.959Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2cef478f-c587-44c1-b2cf-ada768dcee45","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:56.959Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2cef478f-c587-44c1-b2cf-ada768dcee45","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.959Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2cef478f-c587-44c1-b2cf-ada768dcee45"}
{"level":"error","ts":"2026-07-10T11:00:56.959Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"2cef478f-c587-44c1-b2cf-ada768dcee45","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.961Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b5b18fdc-ac6c-4af5-ab02-4e37570d32b4","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:56.961Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b5b18fdc-ac6c-4af5-ab02-4e37570d32b4","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.961Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b5b18fdc-ac6c-4af5-ab02-4e37570d32b4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.961Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e45e6816-cd66-4fb0-9bae-442daa0cb7bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.965Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c7c31eff-e19d-48f9-af57-ca91f0d58165","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.965Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c7c31eff-e19d-48f9-af57-ca91f0d58165","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:56.968Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c7c31eff-e19d-48f9-af57-ca91f0d58165","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:56.968Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c7c31eff-e19d-48f9-af57-ca91f0d58165","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.968Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c7c31eff-e19d-48f9-af57-ca91f0d58165"}
{"level":"error","ts":"2026-07-10T11:00:56.968Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"c7c31eff-e19d-48f9-af57-ca91f0d58165","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:56.972Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b5b18fdc-ac6c-4af5-ab02-4e37570d32b4","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.972Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b5b18fdc-ac6c-4af5-ab02-4e37570d32b4"}
{"level":"error","ts":"2026-07-10T11:00:56.972Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"b5b18fdc-ac6c-4af5-ab02-4e37570d32b4","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.978Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"71ac212c-fca6-4af4-a294-dca83af2be09","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.978Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"71ac212c-fca6-4af4-a294-dca83af2be09","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:56.981Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"71ac212c-fca6-4af4-a294-dca83af2be09","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:56.981Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"71ac212c-fca6-4af4-a294-dca83af2be09","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.981Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"71ac212c-fca6-4af4-a294-dca83af2be09"}
{"level":"error","ts":"2026-07-10T11:00:56.981Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"71ac212c-fca6-4af4-a294-dca83af2be09","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.983Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4591ac0f-717b-44f6-98f3-2a1d487402b3","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:56.983Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4591ac0f-717b-44f6-98f3-2a1d487402b3","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:56.983Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4591ac0f-717b-44f6-98f3-2a1d487402b3","cluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:56.997Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4591ac0f-717b-44f6-98f3-2a1d487402b3","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:56.997Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4591ac0f-717b-44f6-98f3-2a1d487402b3"}
{"level":"error","ts":"2026-07-10T11:00:56.997Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"4591ac0f-717b-44f6-98f3-2a1d487402b3","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.002Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"61917ae1-ae6d-4a57-a88d-a47c02ecdb1e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.002Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"61917ae1-ae6d-4a57-a88d-a47c02ecdb1e","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:57.008Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"61917ae1-ae6d-4a57-a88d-a47c02ecdb1e","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:57.008Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"61917ae1-ae6d-4a57-a88d-a47c02ecdb1e","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.008Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"61917ae1-ae6d-4a57-a88d-a47c02ecdb1e"}
{"level":"error","ts":"2026-07-10T11:00:57.008Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"61917ae1-ae6d-4a57-a88d-a47c02ecdb1e","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.025Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e92f6fba-accc-40fc-a486-85c31b3da5ea","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:57.025Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e92f6fba-accc-40fc-a486-85c31b3da5ea","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.025Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e92f6fba-accc-40fc-a486-85c31b3da5ea","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.048Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3187ab5e-5063-4709-9751-46d0c04abce1","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.048Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3187ab5e-5063-4709-9751-46d0c04abce1","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:57.053Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3187ab5e-5063-4709-9751-46d0c04abce1","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:57.053Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3187ab5e-5063-4709-9751-46d0c04abce1","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.053Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3187ab5e-5063-4709-9751-46d0c04abce1"}
{"level":"error","ts":"2026-07-10T11:00:57.053Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"3187ab5e-5063-4709-9751-46d0c04abce1","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:57.055Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e92f6fba-accc-40fc-a486-85c31b3da5ea","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.055Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e92f6fba-accc-40fc-a486-85c31b3da5ea"}
{"level":"error","ts":"2026-07-10T11:00:57.055Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e92f6fba-accc-40fc-a486-85c31b3da5ea","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.095Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"77ebcd82-c722-479b-ae20-93aabfb8492b","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:57.095Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"77ebcd82-c722-479b-ae20-93aabfb8492b","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.095Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"77ebcd82-c722-479b-ae20-93aabfb8492b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.099Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9e733b88-6e0a-431b-b461-7ebfafc1cb60","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.099Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9e733b88-6e0a-431b-b461-7ebfafc1cb60","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.100Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9d3eaf89-38df-4e18-b452-c1cac8fae4d4"}
{"level":"info","ts":"2026-07-10T11:00:57.100Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"reconcileID":"a572f3ae-4bd7-433d-84f1-fddee1cbfa04"}
{"level":"error","ts":"2026-07-10T11:00:57.102Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9e733b88-6e0a-431b-b461-7ebfafc1cb60","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:57.102Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9e733b88-6e0a-431b-b461-7ebfafc1cb60","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.102Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9e733b88-6e0a-431b-b461-7ebfafc1cb60"}
{"level":"error","ts":"2026-07-10T11:00:57.102Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"9e733b88-6e0a-431b-b461-7ebfafc1cb60","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:57.104Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"77ebcd82-c722-479b-ae20-93aabfb8492b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:285\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:57.104Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"77ebcd82-c722-479b-ae20-93aabfb8492b","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.104Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"77ebcd82-c722-479b-ae20-93aabfb8492b"}
{"level":"error","ts":"2026-07-10T11:00:57.104Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"77ebcd82-c722-479b-ae20-93aabfb8492b","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.104Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:57.104Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.104Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.104Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","namespace":"test-ns-d4rh6","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-10T11:00:57.109Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","name":"kuberay-gateway-access","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T11:00:57.115Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","name":"test-ns-d4rh6-ray-cluster","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-10T11:00:57.122Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","configMap":"kube-rbac-proxy-config-ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.129Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.129Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T11:00:57.134Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8f1cff7e-f1ae-4cf6-a42f-43e695d80275","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.134Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8f1cff7e-f1ae-4cf6-a42f-43e695d80275","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-10T11:00:57.137Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8f1cff7e-f1ae-4cf6-a42f-43e695d80275","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:00:57.137Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8f1cff7e-f1ae-4cf6-a42f-43e695d80275","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.137Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8f1cff7e-f1ae-4cf6-a42f-43e695d80275"}
{"level":"error","ts":"2026-07-10T11:00:57.137Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"8f1cff7e-f1ae-4cf6-a42f-43e695d80275","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-d4rh6 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.139Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a99e9990-962b-4ce6-8ec9-5f37129992b8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.139Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1c2b2ebc-20e5-40fe-b623-ed4ebbb9a2a2","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:57.139Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1c2b2ebc-20e5-40fe-b623-ed4ebbb9a2a2"}
{"level":"info","ts":"2026-07-10T11:00:57.139Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1c2b2ebc-20e5-40fe-b623-ed4ebbb9a2a2","namespace":"test-ns-d4rh6","authClusters":0}
{"level":"info","ts":"2026-07-10T11:00:57.139Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"1c2b2ebc-20e5-40fe-b623-ed4ebbb9a2a2","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T11:00:57.139Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e2c15948-1f95-4aba-947f-5914ba3bad76","namespace":"test-ns-d4rh6","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.140Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"34992bf2-080d-4cac-82e9-97fd54e5a2e4"}
{"level":"error","ts":"2026-07-10T11:00:57.148Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e2c15948-1f95-4aba-947f-5914ba3bad76","issuer":"ray-selfsigned-issuer-ray-cluster","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.148Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e2c15948-1f95-4aba-947f-5914ba3bad76","errorCount":1}
{"level":"info","ts":"2026-07-10T11:00:57.148Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e2c15948-1f95-4aba-947f-5914ba3bad76"}
{"level":"error","ts":"2026-07-10T11:00:57.148Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"e2c15948-1f95-4aba-947f-5914ba3bad76","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:00:57.184Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ada6e8c4-4ad0-45b2-9843-2c9b3883c540","namespacedName":{"name":"ray-cluster","namespace":"test-ns-d4rh6"}}
{"level":"info","ts":"2026-07-10T11:00:57.184Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ada6e8c4-4ad0-45b2-9843-2c9b3883c540"}
{"level":"info","ts":"2026-07-10T11:00:57.184Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ada6e8c4-4ad0-45b2-9843-2c9b3883c540","namespace":"test-ns-d4rh6","authClusters":0}
{"level":"info","ts":"2026-07-10T11:00:57.184Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"ada6e8c4-4ad0-45b2-9843-2c9b3883c540","namespace":"test-ns-d4rh6"}
{"level":"info","ts":"2026-07-10T11:00:57.458Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a967a449-5cfc-48b6-a273-06092942f31b","namespace":"test-ns-d4rh6","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:00:57.458Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-d4rh6"},"namespace":"test-ns-d4rh6","name":"ray-cluster","reconcileID":"a967a449-5cfc-48b6-a273-06092942f31b","namespace":"test-ns-d4rh6","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","podName":"ray-cluster-head-wvb8f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:26.444Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e306e45-c482-4dd7-8bd7-e5f47325f67c","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8"}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8"}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","podName":"ray-cluster-head-wvb8f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:00:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","podName":"ray-cluster-head-wvb8f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:00:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T11:01:27.457Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-10T11:01:27.458Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"2d5987b8-3b21-4d72-9e19-daa6ad6de1f8","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:00:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:01:27.472Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8700395d-c9d7-4778-8b90-7d2199a70398","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:27.472Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"fa71ad44-320f-4854-905f-7e402d58960f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:27.473Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"fa71ad44-320f-4854-905f-7e402d58960f","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:27.473Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"fa71ad44-320f-4854-905f-7e402d58960f","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:01:27.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8700395d-c9d7-4778-8b90-7d2199a70398","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:27.635Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8700395d-c9d7-4778-8b90-7d2199a70398","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:27.635Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8700395d-c9d7-4778-8b90-7d2199a70398","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740"}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740"}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:29.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:29.527Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","podName":"ray-cluster-head-wvb8f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:00:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T11:01:29.527Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","podName":"ray-cluster-head-wvb8f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:00:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-10T11:01:29.527Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:29.527Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fe074a0a-2136-4a31-a738-114208eaf740","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd"}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd"}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-10T11:01:43.480Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"533b4c4f-08d8-4248-8d32-df2c1fc849bd","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:43Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-10T11:01:43.492Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"df34cb36-7540-439a-95ad-78f98f176794","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:43.492Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0e5913c6-2588-40df-a1bd-49c5dce4398c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:43.492Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"df34cb36-7540-439a-95ad-78f98f176794","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:43.492Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"df34cb36-7540-439a-95ad-78f98f176794","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:01:43.501Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0e5913c6-2588-40df-a1bd-49c5dce4398c","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:43.512Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0e5913c6-2588-40df-a1bd-49c5dce4398c","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:43.512Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0e5913c6-2588-40df-a1bd-49c5dce4398c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:43.835Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8a64d225-9dea-47d8-a1f5-21f5f30b7cb4","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:43.835Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8a64d225-9dea-47d8-a1f5-21f5f30b7cb4","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:43.835Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8a64d225-9dea-47d8-a1f5-21f5f30b7cb4","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd"}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd"}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:45.492Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"98cdb915-cb99-44a8-9596-65db0d3ffabd","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:49.408Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.408Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pxgtj"}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"173580c8-0835-4a04-9def-7fdb3d3f9433","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"366fabba-5559-499d-afa7-fc86ad680605","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"173580c8-0835-4a04-9def-7fdb3d3f9433","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"173580c8-0835-4a04-9def-7fdb3d3f9433","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299"}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299"}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"cec530c0-3b2f-4f71-a828-4bf2b6ceb217","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.419Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"cec530c0-3b2f-4f71-a828-4bf2b6ceb217","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"cec530c0-3b2f-4f71-a828-4bf2b6ceb217","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299"}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T11:01:49.420Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","enabled":true}
{"level":"info","ts":"2026-07-10T11:01:49.430Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"366fabba-5559-499d-afa7-fc86ad680605","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:49.432Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"173580c8-0835-4a04-9def-7fdb3d3f9433","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.432Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"27bd40cc-d933-4331-91b2-2a2d1eda3467","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.432Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"27bd40cc-d933-4331-91b2-2a2d1eda3467","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:49.432Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"27bd40cc-d933-4331-91b2-2a2d1eda3467","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:01:49.437Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","name":"ray-cluster-small-group-worker-c9l44"}
{"level":"info","ts":"2026-07-10T11:01:49.437Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:01:49.437Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:43Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:49Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-10T11:01:49.444Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"366fabba-5559-499d-afa7-fc86ad680605","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:49.444Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"366fabba-5559-499d-afa7-fc86ad680605","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.444Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"71f5dd78-d345-4939-b30a-909ea3f6ede6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299"}
{"level":"error","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"535c16fe-e3d3-47ee-900e-0f531da5a299","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:01:49.445Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"61898518-247a-470d-b75c-dcf6e0f0d1fb","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:43Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:49Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311"}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311"}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:43Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:49Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-10T11:01:49.456Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"7d654394-79bb-4466-879e-10b6ab7fbd93","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.457Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"7d654394-79bb-4466-879e-10b6ab7fbd93","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:49.457Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"7d654394-79bb-4466-879e-10b6ab7fbd93","rayCluster":"ray-cluster","podIPs":["10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:01:49.459Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"71f5dd78-d345-4939-b30a-909ea3f6ede6","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311"}
{"level":"error","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7f92091c-3261-4910-81e6-ae1b01131311","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"04294c9c-8875-416f-8552-cd27f462d67b","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:49.525Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:49.526Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"9c7a9803-5849-4c44-add2-7ff4b1426169","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8"}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8"}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:49.530Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"67d66e1d-337e-49ca-b1a4-79c9c39850c8","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:49.531Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"71f5dd78-d345-4939-b30a-909ea3f6ede6","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:49.531Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"71f5dd78-d345-4939-b30a-909ea3f6ede6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.531Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"011be416-9c81-445e-ba70-56e143c21ef4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.547Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"011be416-9c81-445e-ba70-56e143c21ef4","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:49.556Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"011be416-9c81-445e-ba70-56e143c21ef4","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:49.557Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"011be416-9c81-445e-ba70-56e143c21ef4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:49.890Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"af449df8-32ca-45ce-abac-b7bcf28beb15","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51"}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51"}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:50.364Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"467f4f88-7c58-41f9-b831-7721a042be51","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:54.540Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.540Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pxgtj"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"40307206-553d-4ec9-83e5-8d33f30e7456","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ddb43cec-6ac4-45bc-bf98-cb46aaddc822","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6c827fc3-41aa-4f3d-b9c4-bd666b05e076","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ddb43cec-6ac4-45bc-bf98-cb46aaddc822","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ddb43cec-6ac4-45bc-bf98-cb46aaddc822","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"40307206-553d-4ec9-83e5-8d33f30e7456","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"40307206-553d-4ec9-83e5-8d33f30e7456","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8"}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-10T11:01:54.551Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","enabled":true}
{"level":"info","ts":"2026-07-10T11:01:54.562Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6c827fc3-41aa-4f3d-b9c4-bd666b05e076","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:54.563Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ddb43cec-6ac4-45bc-bf98-cb46aaddc822","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.566Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","name":"ray-cluster-small-group-worker-85qpq"}
{"level":"info","ts":"2026-07-10T11:01:54.566Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:54.566Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:01:54.566Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:49Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:54Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T11:01:54.569Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"40307206-553d-4ec9-83e5-8d33f30e7456","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pxgtj.svc","ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:01:54.574Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:01:54.574Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8"}
{"level":"error","ts":"2026-07-10T11:01:54.574Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d1fc9435-d4ca-4dda-95ce-5a914bf21bb8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:01:54.578Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6c827fc3-41aa-4f3d-b9c4-bd666b05e076","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:54.579Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"6c827fc3-41aa-4f3d-b9c4-bd666b05e076","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.579Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a99d12df-01d3-44cc-9351-dde26dda36bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.585Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"40307206-553d-4ec9-83e5-8d33f30e7456","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pxgtj.svc","ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pxgtj.svc","*.ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*-worker-*.test-ns-pxgtj.svc","*-worker-*.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:01:54.585Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"40307206-553d-4ec9-83e5-8d33f30e7456"}
{"level":"info","ts":"2026-07-10T11:01:54.585Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"474de7f4-e10b-476a-a289-29e7c1596c1c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"474de7f4-e10b-476a-a289-29e7c1596c1c","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:54.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"474de7f4-e10b-476a-a289-29e7c1596c1c","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","1 head service found":"ray-cluster-head-svc"}
{"level":"error","ts":"2026-07-10T11:01:54.625Z","msg":"Failed to update certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"474de7f4-e10b-476a-a289-29e7c1596c1c","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).updateCertificateWithPodIPs\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:465\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:212\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T11:01:54.625Z","msg":"Failed to update worker certificate with pod IPs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"474de7f4-e10b-476a-a289-29e7c1596c1c","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:213\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"474de7f4-e10b-476a-a289-29e7c1596c1c"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"error","ts":"2026-07-10T11:01:54.625Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"474de7f4-e10b-476a-a289-29e7c1596c1c","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:01:54.625Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ca41805f-8263-4a90-9c8b-e1f03b80ef10","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:49Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:54Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T11:01:54.631Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c2a63044-6fd3-4841-bc9c-f294ae4505c6","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.631Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c2a63044-6fd3-4841-bc9c-f294ae4505c6","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:54.631Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c2a63044-6fd3-4841-bc9c-f294ae4505c6","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:01:54.631Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c2a63044-6fd3-4841-bc9c-f294ae4505c6"}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a99d12df-01d3-44cc-9351-dde26dda36bc","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26"}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26"}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9a582a9e-362b-4072-aed7-c486a76c534d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9a582a9e-362b-4072-aed7-c486a76c534d","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9a582a9e-362b-4072-aed7-c486a76c534d","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9a582a9e-362b-4072-aed7-c486a76c534d"}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","podName":"ray-cluster-small-group-worker-85qpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:01:54.726Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:49Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:54Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T11:01:54.733Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a99d12df-01d3-44cc-9351-dde26dda36bc","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:54.733Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a99d12df-01d3-44cc-9351-dde26dda36bc","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.733Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"bced32bf-b06f-495d-a05a-57917485def4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:54.734Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:01:54.734Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26"}
{"level":"error","ts":"2026-07-10T11:01:54.734Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c2fce0dc-7a99-42cc-ad02-c9a942f9ce26","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:01:54.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659"}
{"level":"info","ts":"2026-07-10T11:01:54.739Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659"}
{"level":"info","ts":"2026-07-10T11:01:54.739Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","podName":"ray-cluster-small-group-worker-85qpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:54.740Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"1fd05450-b38b-4b0d-b954-5de274b51659","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:54.744Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"bced32bf-b06f-495d-a05a-57917485def4","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:01:54.752Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"bced32bf-b06f-495d-a05a-57917485def4","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:01:54.752Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"bced32bf-b06f-495d-a05a-57917485def4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:01:55.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955"}
{"level":"info","ts":"2026-07-10T11:01:55.021Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955"}
{"level":"info","ts":"2026-07-10T11:01:55.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:55.022Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"e44ad17c-52bd-4e26-9db9-11acebf4f955","seconds":300}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3"}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3"}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:01:55.548Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:01:55.549Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:01:55.549Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"b53dc105-0c3d-45d4-bceb-bc5198c00cc3","seconds":300}
{"level":"info","ts":"2026-07-10T11:02:24.632Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f895ac1b-c0fe-40df-84a0-429f409c6559","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:02:24.632Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f895ac1b-c0fe-40df-84a0-429f409c6559","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:02:24.632Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f895ac1b-c0fe-40df-84a0-429f409c6559","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:02:24.632Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f895ac1b-c0fe-40df-84a0-429f409c6559","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:02:24.645Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f895ac1b-c0fe-40df-84a0-429f409c6559","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pxgtj.svc","ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.37","10.134.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:02:24.663Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f895ac1b-c0fe-40df-84a0-429f409c6559","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pxgtj.svc","ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pxgtj.svc","*.ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*-worker-*.test-ns-pxgtj.svc","*-worker-*.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.37","10.134.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:02:24.663Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f895ac1b-c0fe-40df-84a0-429f409c6559","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c"}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c"}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:00.639Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"687385ea-8a27-4bc7-a804-3b623ee3bd6c","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6"}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6"}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:01.644Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:03:01.645Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:01.645Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bc800ce7-697d-404d-b09f-6075ad6750b6","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6"}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6"}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:11.684Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"579b24ab-6f63-441c-8cd8-c0982b2a2ae6","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72"}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72"}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:12.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:12.691Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:01:49Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T11:03:12.691Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:03:12.691Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3ac991e6-d957-43ce-a6fa-019c5907bc72","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:01:54Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:12Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T11:03:12.702Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c5171882-21b8-426d-8fee-cec10c4ac993","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:12.702Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53c062fc-d6f8-4308-8e9e-fde193c1db81","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:12.702Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c5171882-21b8-426d-8fee-cec10c4ac993","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:03:12.702Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c5171882-21b8-426d-8fee-cec10c4ac993","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:03:12.702Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c5171882-21b8-426d-8fee-cec10c4ac993","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:03:12.702Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c5171882-21b8-426d-8fee-cec10c4ac993","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:03:12.718Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53c062fc-d6f8-4308-8e9e-fde193c1db81","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:03:12.727Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53c062fc-d6f8-4308-8e9e-fde193c1db81","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:03:12.727Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53c062fc-d6f8-4308-8e9e-fde193c1db81","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3"}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3"}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:01:49Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:14.702Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"6e2e4555-ce33-45e9-9593-78e625337cd3","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:15.853Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d"}
{"level":"info","ts":"2026-07-10T11:03:15.853Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d"}
{"level":"info","ts":"2026-07-10T11:03:15.853Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:01:49Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:15.854Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"bfb6dcf0-2e33-416a-89e6-49615b19613d","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3"}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3"}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:16.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:16.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","podName":"ray-cluster-small-group-worker-c9l44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:01:49Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T11:03:16.926Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:16.926Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"83b4c6ea-608b-4cd1-b2d5-dd2d48b1b2e3","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075"}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075"}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","podName":"ray-cluster-small-group-worker-85qpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:03:23.719Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5be526b4-b348-4df8-8b66-6f84d4fac075","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:12Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:23Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T11:03:23.731Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9c4b6922-c57b-40f2-a5f5-c720f97a6d4f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:23.731Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5de050ae-2387-4305-af91-061e8b5bb4df","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:23.731Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9c4b6922-c57b-40f2-a5f5-c720f97a6d4f","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:03:23.731Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9c4b6922-c57b-40f2-a5f5-c720f97a6d4f","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:03:23.731Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9c4b6922-c57b-40f2-a5f5-c720f97a6d4f","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:03:23.731Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"9c4b6922-c57b-40f2-a5f5-c720f97a6d4f","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:03:23.742Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5de050ae-2387-4305-af91-061e8b5bb4df","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:03:23.752Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5de050ae-2387-4305-af91-061e8b5bb4df","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:03:23.752Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"5de050ae-2387-4305-af91-061e8b5bb4df","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:24.664Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"152d7eed-5966-48c0-a978-549cb8bced20","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:24.665Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"152d7eed-5966-48c0-a978-549cb8bced20","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:03:24.665Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"152d7eed-5966-48c0-a978-549cb8bced20","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:03:24.665Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"152d7eed-5966-48c0-a978-549cb8bced20","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:03:24.665Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"152d7eed-5966-48c0-a978-549cb8bced20","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78"}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78"}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","podName":"ray-cluster-small-group-worker-85qpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:25.732Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d6e9a933-45b4-4c9c-b0bc-4993c3857d78","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42"}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42"}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:26.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:26.889Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","podName":"ray-cluster-small-group-worker-85qpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-10T11:03:26.889Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:26.889Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"31b1ee17-024a-4e69-83d6-a3417beb5b42","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709"}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709"}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:27.891Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","podName":"ray-cluster-small-group-worker-85qpq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:01:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T11:03:27.892Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:03:27.892Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"cf1b9899-a27a-46de-9dbd-f2f168479709","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:23Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:27Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T11:03:27.903Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0065186c-2539-4a4b-be99-5230cd7f9718","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:27.903Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f4c0eb2a-2313-473f-bff5-3aa4326691f0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:27.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0065186c-2539-4a4b-be99-5230cd7f9718","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:03:27.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0065186c-2539-4a4b-be99-5230cd7f9718","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:03:27.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0065186c-2539-4a4b-be99-5230cd7f9718","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:03:27.904Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0065186c-2539-4a4b-be99-5230cd7f9718","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:03:27.920Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f4c0eb2a-2313-473f-bff5-3aa4326691f0","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:03:27.930Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f4c0eb2a-2313-473f-bff5-3aa4326691f0","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:03:27.930Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f4c0eb2a-2313-473f-bff5-3aa4326691f0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560"}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560"}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:29.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","podName":"ray-cluster-small-group-worker-85qpq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-10T11:01:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-10T11:03:29.905Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:29.905Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"fd11058c-ca31-4c2c-bfca-a63258eeb560","seconds":300}
{"level":"info","ts":"2026-07-10T11:03:38.912Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc"}
{"level":"info","ts":"2026-07-10T11:03:38.912Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc"}
{"level":"info","ts":"2026-07-10T11:03:38.912Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:38.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:38.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:38.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:38.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:38.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:38.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:38.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:38.913Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:38.913Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:03:38.913Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"7c5d1986-bd78-404d-be91-358f0d8516bc","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:27Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:38Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-10T11:03:38.925Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"768cb46f-a1b4-482f-9a34-b8bde2474f19","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:38.925Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b501c418-55de-43ae-81f2-2ff49e384e24","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:38.925Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"768cb46f-a1b4-482f-9a34-b8bde2474f19","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:03:38.925Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"768cb46f-a1b4-482f-9a34-b8bde2474f19","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:03:38.925Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"768cb46f-a1b4-482f-9a34-b8bde2474f19","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:03:38.925Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"768cb46f-a1b4-482f-9a34-b8bde2474f19","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:03:38.937Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b501c418-55de-43ae-81f2-2ff49e384e24","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:03:38.945Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b501c418-55de-43ae-81f2-2ff49e384e24","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:03:38.945Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b501c418-55de-43ae-81f2-2ff49e384e24","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2"}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2"}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:03:40.925Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"711459e3-4245-4d63-9a7c-e19db13346b2","seconds":300}
{"level":"info","ts":"2026-07-10T11:04:24.666Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64814ae7-84dd-458f-acfa-1d2f2f5483a9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:24.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64814ae7-84dd-458f-acfa-1d2f2f5483a9","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:04:24.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64814ae7-84dd-458f-acfa-1d2f2f5483a9","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:04:24.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64814ae7-84dd-458f-acfa-1d2f2f5483a9","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:04:24.666Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64814ae7-84dd-458f-acfa-1d2f2f5483a9","rayCluster":"ray-cluster","podIPs":["10.134.0.19","10.133.0.53","10.132.0.37"]}
{"level":"info","ts":"2026-07-10T11:04:36.725Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:36.726Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pxgtj"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0d2196d7-11f0-4d26-9b15-98b6bf53da76","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"83292380-41d8-4a31-9954-01b0dccbe5da","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"4f7f637f-2918-4d59-acb6-41bbd98d4011","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0d2196d7-11f0-4d26-9b15-98b6bf53da76","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0d2196d7-11f0-4d26-9b15-98b6bf53da76","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"4f7f637f-2918-4d59-acb6-41bbd98d4011","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"4f7f637f-2918-4d59-acb6-41bbd98d4011","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"4f7f637f-2918-4d59-acb6-41bbd98d4011","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"4f7f637f-2918-4d59-acb6-41bbd98d4011","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","worker Pod":"ray-cluster-small-group-worker-85qpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-85qpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:04:36.739Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","namespace":"test-ns-pxgtj","name":"ray-cluster-small-group-worker-85qpq"}
{"level":"info","ts":"2026-07-10T11:04:36.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:04:36.749Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:04:36.825Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:04:36.825Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:38Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:36Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T11:04:36.825Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0d2196d7-11f0-4d26-9b15-98b6bf53da76","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:36.833Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:04:36.833Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b"}
{"level":"error","ts":"2026-07-10T11:04:36.833Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"5ae048eb-f696-4655-a3dd-c7fe6999de8b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:04:36.925Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8d6da56d-987e-4995-aef7-04b2bfbd7e36","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8d6da56d-987e-4995-aef7-04b2bfbd7e36","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8d6da56d-987e-4995-aef7-04b2bfbd7e36","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8d6da56d-987e-4995-aef7-04b2bfbd7e36","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:04:37.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:37.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:04:37.026Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8d6da56d-987e-4995-aef7-04b2bfbd7e36","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:04:37.026Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:04:37.026Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:38Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T11:04:37.030Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"83292380-41d8-4a31-9954-01b0dccbe5da","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf"}
{"level":"error","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"767e321a-893e-4374-bd8d-500fe909c9bf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad"}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad"}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:04:37.033Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"96a8be1e-22b6-42a1-bde6-1e32a2017bad","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:38Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T11:04:37.125Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"83292380-41d8-4a31-9954-01b0dccbe5da","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:04:37.125Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"83292380-41d8-4a31-9954-01b0dccbe5da","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.125Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0f3e4340-ca56-4ee2-be7c-d4654040773f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0f3e4340-ca56-4ee2-be7c-d4654040773f","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"e3c2b6b4-d3f3-4c7f-a1f7-60aee12bf5e2","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"e3c2b6b4-d3f3-4c7f-a1f7-60aee12bf5e2","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"e3c2b6b4-d3f3-4c7f-a1f7-60aee12bf5e2","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"e3c2b6b4-d3f3-4c7f-a1f7-60aee12bf5e2","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"e3c2b6b4-d3f3-4c7f-a1f7-60aee12bf5e2","rayCluster":"ray-cluster","podIPs":["10.132.0.37","10.134.0.19","10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:04:37.136Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:04:37.137Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:03:38Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T11:04:37.143Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:04:37.143Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba"}
{"level":"error","ts":"2026-07-10T11:04:37.143Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"76c3a236-b012-4ffe-843b-b181907367ba","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:04:37.144Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0f3e4340-ca56-4ee2-be7c-d4654040773f","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:04:37.144Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"0f3e4340-ca56-4ee2-be7c-d4654040773f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.144Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f8b93139-6471-435c-b103-a22c83b22043","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.148Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314"}
{"level":"info","ts":"2026-07-10T11:04:37.148Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314"}
{"level":"info","ts":"2026-07-10T11:04:37.148Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:37.148Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:37.148Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:37.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:04:37.149Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:04:37.149Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"d31c4d17-e9d6-427d-be49-6b27a25d2314","seconds":300}
{"level":"info","ts":"2026-07-10T11:04:37.153Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f8b93139-6471-435c-b103-a22c83b22043","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:04:37.162Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f8b93139-6471-435c-b103-a22c83b22043","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:04:37.162Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"f8b93139-6471-435c-b103-a22c83b22043","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506"}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506"}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:04:37.319Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"884ed948-2c64-462e-af90-179f1854f506","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-10T11:04:37.332Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"973d85ce-ef4d-4a5b-85b6-20cbf1a98453","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.332Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64008948-52e7-4fd6-82f9-63507f5c51c0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:37.332Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64008948-52e7-4fd6-82f9-63507f5c51c0","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:04:37.332Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64008948-52e7-4fd6-82f9-63507f5c51c0","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:04:37.332Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64008948-52e7-4fd6-82f9-63507f5c51c0","pod":"ray-cluster-small-group-worker-85qpq","ip":"10.134.0.19"}
{"level":"info","ts":"2026-07-10T11:04:37.332Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"64008948-52e7-4fd6-82f9-63507f5c51c0","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37","10.134.0.19"]}
{"level":"info","ts":"2026-07-10T11:04:37.343Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"973d85ce-ef4d-4a5b-85b6-20cbf1a98453","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:04:37.355Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"973d85ce-ef4d-4a5b-85b6-20cbf1a98453","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:04:37.355Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"973d85ce-ef4d-4a5b-85b6-20cbf1a98453","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:04:38.140Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5"}
{"level":"info","ts":"2026-07-10T11:04:38.140Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5"}
{"level":"info","ts":"2026-07-10T11:04:38.140Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:38.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:38.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:38.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:04:38.141Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:04:38.141Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"544e9dd2-10d1-4a8f-8ff8-7325062130f5","seconds":300}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4"}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4"}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:04:38.143Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","namespace":"test-ns-pxgtj","name":"ray-cluster-small-group-worker-85qpq"}
{"level":"info","ts":"2026-07-10T11:04:38.146Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","The worker Pod has already been deleted":"ray-cluster-small-group-worker-85qpq"}
{"level":"info","ts":"2026-07-10T11:04:38.146Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:04:38.146Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:04:38.146Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:04:38.147Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"ce950dda-2565-4dfb-9230-fc3cafd481f4","seconds":300}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef"}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef"}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:04:39.333Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","namespace":"test-ns-pxgtj","name":"ray-cluster-small-group-worker-85qpq"}
{"level":"info","ts":"2026-07-10T11:04:39.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","The worker Pod has already been deleted":"ray-cluster-small-group-worker-85qpq"}
{"level":"info","ts":"2026-07-10T11:04:39.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-10T11:04:39.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:04:39.338Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:04:39.338Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"8e73151d-c3ae-464b-8fca-ba316d3e9aef","seconds":300}
{"level":"info","ts":"2026-07-10T11:05:24.666Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"87aae92f-a9fb-43a3-9317-741a3595e171","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:24.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"87aae92f-a9fb-43a3-9317-741a3595e171","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:05:24.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"87aae92f-a9fb-43a3-9317-741a3595e171","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:05:24.693Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"87aae92f-a9fb-43a3-9317-741a3595e171","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-pxgtj.svc","ray-cluster-head-svc.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:05:24.706Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"87aae92f-a9fb-43a3-9317-741a3595e171","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-pxgtj.svc","ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-pxgtj.svc","*.ray-cluster-worker-svc.test-ns-pxgtj.svc.cluster.local","*-worker-*.test-ns-pxgtj.svc","*-worker-*.test-ns-pxgtj.svc.cluster.local"],"ipAddresses":["10.133.0.53","10.132.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-10T11:05:24.706Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"87aae92f-a9fb-43a3-9317-741a3595e171","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37"]}
{"level":"info","ts":"2026-07-10T11:05:47.689Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.689Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-pxgtj"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53e3a013-1bea-4396-93bc-dd9acaf317f0","namespacedName":{"name":"ray-cluster","namespace":"test-ns-pxgtj"}}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c4cfdcee-f923-46ce-9b7b-a7edec691578","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53e3a013-1bea-4396-93bc-dd9acaf317f0","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53e3a013-1bea-4396-93bc-dd9acaf317f0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"91304099-362a-4a04-8bea-6027ec5ab80e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c4cfdcee-f923-46ce-9b7b-a7edec691578","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c4cfdcee-f923-46ce-9b7b-a7edec691578","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"c4cfdcee-f923-46ce-9b7b-a7edec691578","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37"]}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","worker Pod":"ray-cluster-small-group-worker-c9l44","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-c9l44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:05:47.700Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","namespace":"test-ns-pxgtj","name":"ray-cluster-small-group-worker-c9l44"}
{"level":"info","ts":"2026-07-10T11:05:47.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:05:47.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:05:47.713Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"53e3a013-1bea-4396-93bc-dd9acaf317f0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.713Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"438d3250-d336-42fd-b171-909ac0d76ae3","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.725Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"438d3250-d336-42fd-b171-909ac0d76ae3","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:05:47.725Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"438d3250-d336-42fd-b171-909ac0d76ae3","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:05:47.725Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"91304099-362a-4a04-8bea-6027ec5ab80e","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:05:47.725Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"438d3250-d336-42fd-b171-909ac0d76ae3","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37"]}
{"level":"info","ts":"2026-07-10T11:05:47.725Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:05:47.726Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:05:47Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:05:47.825Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:05:47.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835"}
{"level":"error","ts":"2026-07-10T11:05:47.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"3bbc160b-f606-4ba8-b686-b5037f24c835","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:05:47.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e"}
{"level":"info","ts":"2026-07-10T11:05:47.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e"}
{"level":"info","ts":"2026-07-10T11:05:47.826Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:05:47.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:05:47.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:47.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:05:47.826Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:05:47.826Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"423ff2e9-fd8f-4005-89e9-3c6694887e6e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:05:47Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:05:47.925Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"91304099-362a-4a04-8bea-6027ec5ab80e","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:05:47.925Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"91304099-362a-4a04-8bea-6027ec5ab80e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.925Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b9c64d70-f724-49ca-8462-b6c316281f4c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.937Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b9c64d70-f724-49ca-8462-b6c316281f4c","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:05:47.937Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c"}
{"level":"info","ts":"2026-07-10T11:05:47.937Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c"}
{"level":"info","ts":"2026-07-10T11:05:47.944Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a73d9d42-f745-4157-9022-c6accb9bd13b","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:47.944Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a73d9d42-f745-4157-9022-c6accb9bd13b","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:05:47.944Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a73d9d42-f745-4157-9022-c6accb9bd13b","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:05:47.944Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"a73d9d42-f745-4157-9022-c6accb9bd13b","rayCluster":"ray-cluster","podIPs":["10.132.0.37","10.133.0.53"]}
{"level":"info","ts":"2026-07-10T11:05:47.938Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b9c64d70-f724-49ca-8462-b6c316281f4c","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"b9c64d70-f724-49ca-8462-b6c316281f4c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ea8490cb-1e1e-4416-9ce9-b3542ee742e8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:05:48.025Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:04:37Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:05:48Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:05:48.033Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-pxgtj","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-10T11:05:48.033Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c"}
{"level":"error","ts":"2026-07-10T11:05:48.033Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"c4eaad6b-2641-4d11-af9f-e9a0b210fc3c","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T11:05:48.037Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ea8490cb-1e1e-4416-9ce9-b3542ee742e8","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524"}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524"}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:05:48.040Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"aa99fec6-3895-4a15-a92f-2ad13d6dc524","seconds":300}
{"level":"info","ts":"2026-07-10T11:05:48.047Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ea8490cb-1e1e-4416-9ce9-b3542ee742e8","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-10T11:05:48.047Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"ea8490cb-1e1e-4416-9ce9-b3542ee742e8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:48.283Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920"}
{"level":"info","ts":"2026-07-10T11:05:48.283Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920"}
{"level":"info","ts":"2026-07-10T11:05:48.283Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:05:48.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:05:48.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:48.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:05:48.284Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-10T11:05:48.284Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"a5513859-facb-458a-b62b-40f719b6d920","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:05:47Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-10T11:05:48Z","stateTransitionTimes":{"ready":"2026-07-10T11:01:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.53","serviceIP":"10.133.0.53","podName":"ray-cluster-head-wvb8f","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-10T11:00:14Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-10T11:01:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-10T11:00:14Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-10T11:05:48.295Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"00236c92-624a-479a-8dd2-0832572f6030","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:48.295Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8a344ae6-9255-4e93-970f-2064803b4b56","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:48.295Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"00236c92-624a-479a-8dd2-0832572f6030","pod":"ray-cluster-head-wvb8f","ip":"10.133.0.53"}
{"level":"info","ts":"2026-07-10T11:05:48.295Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"00236c92-624a-479a-8dd2-0832572f6030","pod":"ray-cluster-small-group-worker-c9l44","ip":"10.132.0.37"}
{"level":"info","ts":"2026-07-10T11:05:48.295Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"00236c92-624a-479a-8dd2-0832572f6030","rayCluster":"ray-cluster","podIPs":["10.133.0.53","10.132.0.37"]}
{"level":"info","ts":"2026-07-10T11:05:48.300Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"namespace":"test-ns-pxgtj","name":"ray-cluster","reconcileID":"8a344ae6-9255-4e93-970f-2064803b4b56","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb"}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb"}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:05:48.599Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"77620dee-4620-4a72-b762-cd31342909cb","seconds":300}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5"}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5"}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","Found 1 head Pod":"ray-cluster-head-wvb8f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","head Pod":"ray-cluster-head-wvb8f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-wvb8f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-10T11:05:48.602Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","namespace":"test-ns-pxgtj","name":"ray-cluster-small-group-worker-c9l44"}
{"level":"info","ts":"2026-07-10T11:05:48.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","The worker Pod has already been deleted":"ray-cluster-small-group-worker-c9l44"}
{"level":"info","ts":"2026-07-10T11:05:48.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-10T11:05:48.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-10T11:05:48.605Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-10T11:05:48.605Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-pxgtj"},"reconcileID":"51fb89fa-1783-430e-ace3-f5b925beeda5","seconds":300}